Adding simple memory usage test

* improving unique key fix

* also bump timestamp along 1 sec

.github/workflows/containers.yml

@@ -88,7 +88,7 @@ jobs:
       - name: Build and push :dev
         id: docker_build
         if: ${{ github.ref }} == "refs/heads/master"
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./
           file: ./Dockerfile
@@ -106,7 +106,7 @@ jobs:
       - name: Build and push :tag
         id: docker_build_tag_release
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./
           file: ./Dockerfile

.github/workflows/test-container-build.yml

@@ -51,7 +51,7 @@ jobs:
         # Check we can still build under alpine/musl
         - name: Test that the docker containers can build (musl via alpine check)
           id: docker_build_musl
-          uses: docker/build-push-action@v5
+          uses: docker/build-push-action@v6
           with:
             context: ./
             file: ./.github/test/Dockerfile-alpine
@@ -59,7 +59,7 @@ jobs:
 
         - name: Test that the docker containers can build
           id: docker_build
-          uses: docker/build-push-action@v5
+          uses: docker/build-push-action@v6
           # https://github.com/docker/build-push-action#customizing
           with:
             context: ./

.github/workflows/test-only.yml

@@ -4,17 +4,10 @@ name: ChangeDetection.io App Test
 on: [push, pull_request]
 
 jobs:
-  test-application:
+  lint-code:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-
-      # Mainly just for link/flake8
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
-
       - name: Lint with flake8
         run: |
           pip3 install flake8
@@ -23,202 +16,24 @@ jobs:
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
-      - name: Spin up ancillary testable services
-        run: |
-          
-          docker network create changedet-network
-          
-          # Selenium
-          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
-          
-          # SocketPuppetBrowser + Extra for custom browser test
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest                    
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url  -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
-
-      - name: Build changedetection.io container for testing
-        run: |         
-          # Build a changedetection.io container and start testing inside
-          docker build --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
-          # Debug info
-          docker run test-changedetectionio  bash -c 'pip list'
-
-      - name: Spin up ancillary SMTP+Echo message test server
-        run: |
-          # Debug SMTP server/echo message back server
-          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'python changedetectionio/tests/smtp/smtp-test-server.py' 
-
-      - name: Show docker container state and other debug info
-        run: |
-          set -x
-          echo "Running processes in docker..."
-          docker ps
-
-      - name: Test built container with Pytest (generally as requests/plaintext fetching)
-        run: |
-          # Unit tests
-          echo "run test with unittest"
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
-          
-          # All tests
-          echo "run test with pytest"
-          # The default pytest logger_level is TRACE
-          # To change logger_level for pytest(test/conftest.py),
-          # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
-          docker run --name test-cdio-basic-tests --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
-
-# PLAYWRIGHT/NODE-> CDP
-      - name: Playwright and SocketPuppetBrowser - Specific tests in built container
-        run: |
-          # Playwright via Sockpuppetbrowser fetch
-          # tests/visualselector/test_fetch_data.py will do browser steps  
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+  test-application-3-10:
+    needs: lint-code
+    uses: ./.github/workflows/test-stack-reusable-workflow.yml
+    with:
+      python-version: '3.10'
 
 
-      - name: Playwright and SocketPuppetBrowser - Headers and requests
-        run: |       
-          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+  test-application-3-11:
+    needs: lint-code
+    uses: ./.github/workflows/test-stack-reusable-workflow.yml
+    with:
+      python-version: '3.11'
+      skip-pypuppeteer: true
 
-      - name: Playwright and SocketPuppetBrowser - Restock detection
-        run: |                            
-          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+  test-application-3-12:
+    needs: lint-code
+    uses: ./.github/workflows/test-stack-reusable-workflow.yml
+    with:
+      python-version: '3.12'
+      skip-pypuppeteer: true
 
-# STRAIGHT TO CDP
-      - name: Pyppeteer and SocketPuppetBrowser - Specific tests in built container
-        run: |
-          # Playwright via Sockpuppetbrowser fetch 
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
-
-      - name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
-        run: |       
-          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
-          docker run --name "changedet" --hostname changedet --rm  -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-
-      - name: Pyppeteer and SocketPuppetBrowser - Restock detection
-        run: |                            
-          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet"  -e "FAST_PUPPETEER_CHROME_FETCHER=True"  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
-
-# SELENIUM
-      - name: Specific tests in built container for Selenium
-        run: |
-          # Selenium fetch
-          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
-
-      - name: Specific tests in built container for headers and requests checks with Selenium
-        run: |
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-
-# OTHER STUFF
-      - name: Test SMTP notification mime types
-        run: |
-          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
-          docker run --rm  --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
-
-      # @todo Add a test via playwright/puppeteer
-      # squid with auth is tested in run_proxy_tests.sh -> tests/proxy_list/test_select_custom_proxy.py
-      - name: Test proxy squid style interaction
-        run: |
-          cd changedetectionio
-          ./run_proxy_tests.sh
-          cd ..
-
-      - name: Test proxy SOCKS5 style interaction
-        run: |
-          cd changedetectionio
-          ./run_socks_proxy_tests.sh
-          cd ..
-
-      - name: Test custom browser URL
-        run: |
-          cd changedetectionio
-          ./run_custom_browser_url_tests.sh
-          cd ..
-
-      - name: Test changedetection.io container starts+runs basically without error
-        run: |
-          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
-          sleep 3
-          # Should return 0 (no error) when grep finds it
-          curl --retry-connrefused --retry 6  -s http://localhost:5556 |grep -q checkbox-uuid
-          
-          # and IPv6
-          curl --retry-connrefused --retry 6  -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
-
-          # Check whether TRACE log is enabled.
-          # Also, check whether TRACE is came from STDERR
-          docker logs test-changedetectionio 2>&1 1>/dev/null | grep 'TRACE log is enabled' || exit 1
-          # Check whether DEBUG is came from STDOUT
-          docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
-
-          docker kill test-changedetectionio
-
-      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
-        run: |
-          
-          echo SIGINT Shutdown request test
-          docker run --name sig-test -d test-changedetectionio
-          sleep 3
-          echo ">>> Sending SIGINT to sig-test container"
-          docker kill --signal=SIGINT sig-test
-          sleep 3
-          # invert the check (it should be not 0/not running)
-          docker ps
-          # check signal catch(STDERR) log. Because of
-          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
-          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
-          test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]
-          then
-            echo "Looks like container was running when it shouldnt be"
-            docker ps
-            exit 1
-          fi
-          
-          # @todo - scan the container log to see the right "graceful shutdown" text exists 
-          docker rm sig-test
-          
-          echo SIGTERM Shutdown request test
-          docker run --name sig-test -d test-changedetectionio
-          sleep 3
-          echo ">>> Sending SIGTERM to sig-test container"
-          docker kill --signal=SIGTERM sig-test
-          sleep 3
-          # invert the check (it should be not 0/not running)
-          docker ps
-          # check signal catch(STDERR) log. Because of
-          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
-          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
-          test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]
-          then
-            echo "Looks like container was running when it shouldnt be"
-            docker ps
-            exit 1
-          fi
-          
-          # @todo - scan the container log to see the right "graceful shutdown" text exists           
-          docker rm sig-test
-
-      - name: Dump container log
-        if: always()
-        run: |
-          mkdir output-logs
-          docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout.txt
-          docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr.txt
-
-      - name: Store container log
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: test-cdio-basic-tests-output
-          path: output-logs

.github/workflows/test-stack-reusable-workflow.yml

@@ -0,0 +1,239 @@
+name: ChangeDetection.io App Test
+
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        description: 'Python version to use'
+        required: true
+        type: string
+        default: '3.10'
+      skip-pypuppeteer:
+        description: 'Skip PyPuppeteer (not supported in 3.11/3.12)'
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  test-application:
+    runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v4
+
+      # Mainly just for link/flake8
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Build changedetection.io container for testing under Python ${{ env.PYTHON_VERSION }}
+        run: |
+          echo "---- Building for Python ${{ env.PYTHON_VERSION }} -----"
+          # Build a changedetection.io container and start testing inside
+          docker build --build-arg PYTHON_VERSION=${{ env.PYTHON_VERSION }} --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
+          # Debug info
+          docker run test-changedetectionio  bash -c 'pip list'         
+
+      - name: We should be Python ${{ env.PYTHON_VERSION }} ...
+        run: |         
+          docker run test-changedetectionio  bash -c 'python3 --version'
+
+      - name: Spin up ancillary testable services
+        run: |
+          
+          docker network create changedet-network
+          
+          # Selenium
+          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
+          
+          # SocketPuppetBrowser + Extra for custom browser test
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest                    
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url  -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
+
+      - name: Spin up ancillary SMTP+Echo message test server
+        run: |
+          # Debug SMTP server/echo message back server
+          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'pip3 install aiosmtpd && python changedetectionio/tests/smtp/smtp-test-server.py'
+          docker ps
+
+      - name: Show docker container state and other debug info
+        run: |
+          set -x
+          echo "Running processes in docker..."
+          docker ps
+
+      - name: Test built container with Pytest (generally as requests/plaintext fetching)
+        run: |
+          # Unit tests
+          echo "run test with unittest"
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
+          
+          # All tests
+          echo "run test with pytest"
+          # The default pytest logger_level is TRACE
+          # To change logger_level for pytest(test/conftest.py),
+          # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
+          docker run --name test-cdio-basic-tests --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
+
+# PLAYWRIGHT/NODE-> CDP
+      - name: Playwright and SocketPuppetBrowser - Specific tests in built container
+        run: |
+          # Playwright via Sockpuppetbrowser fetch
+          # tests/visualselector/test_fetch_data.py will do browser steps  
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+
+
+      - name: Playwright and SocketPuppetBrowser - Headers and requests
+        run: |       
+          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+
+      - name: Playwright and SocketPuppetBrowser - Restock detection
+        run: |                            
+          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
+# STRAIGHT TO CDP
+      - name: Pyppeteer and SocketPuppetBrowser - Specific tests in built container
+        if: ${{ inputs.skip-pypuppeteer == false }}
+        run: |
+          # Playwright via Sockpuppetbrowser fetch 
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+
+      - name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
+        if: ${{ inputs.skip-pypuppeteer == false }}
+        run: |
+          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
+          docker run --name "changedet" --hostname changedet --rm  -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+
+      - name: Pyppeteer and SocketPuppetBrowser - Restock detection
+        if: ${{ inputs.skip-pypuppeteer == false }}
+        run: |                            
+          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet"  -e "FAST_PUPPETEER_CHROME_FETCHER=True"  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
+# SELENIUM
+      - name: Specific tests in built container for Selenium
+        run: |
+          # Selenium fetch
+          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
+
+      - name: Specific tests in built container for headers and requests checks with Selenium
+        run: |
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+
+# OTHER STUFF
+      - name: Test SMTP notification mime types
+        run: |
+          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
+          # "mailserver" hostname defined above
+          docker run --rm --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
+
+      # @todo Add a test via playwright/puppeteer
+      # squid with auth is tested in run_proxy_tests.sh -> tests/proxy_list/test_select_custom_proxy.py
+      - name: Test proxy squid style interaction
+        run: |
+          cd changedetectionio
+          ./run_proxy_tests.sh
+          cd ..
+
+      - name: Test proxy SOCKS5 style interaction
+        run: |
+          cd changedetectionio
+          ./run_socks_proxy_tests.sh
+          cd ..
+
+      - name: Test custom browser URL
+        run: |
+          cd changedetectionio
+          ./run_custom_browser_url_tests.sh
+          cd ..
+
+      - name: Test changedetection.io container starts+runs basically without error
+        run: |
+          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
+          sleep 3
+          # Should return 0 (no error) when grep finds it
+          curl --retry-connrefused --retry 6  -s http://localhost:5556 |grep -q checkbox-uuid
+          
+          # and IPv6
+          curl --retry-connrefused --retry 6  -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+
+          # Check whether TRACE log is enabled.
+          # Also, check whether TRACE is came from STDERR
+          docker logs test-changedetectionio 2>&1 1>/dev/null | grep 'TRACE log is enabled' || exit 1
+          # Check whether DEBUG is came from STDOUT
+          docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
+
+          docker kill test-changedetectionio
+
+      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
+        run: |
+          
+          echo SIGINT Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGINT to sig-test container"
+          docker kill --signal=SIGINT sig-test
+          sleep 3
+          # invert the check (it should be not 0/not running)
+          docker ps
+          # check signal catch(STDERR) log. Because of
+          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
+          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
+          test -z "`docker ps|grep sig-test`"
+          if [ $? -ne 0 ]
+          then
+            echo "Looks like container was running when it shouldnt be"
+            docker ps
+            exit 1
+          fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists 
+          docker rm sig-test
+          
+          echo SIGTERM Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGTERM to sig-test container"
+          docker kill --signal=SIGTERM sig-test
+          sleep 3
+          # invert the check (it should be not 0/not running)
+          docker ps
+          # check signal catch(STDERR) log. Because of
+          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
+          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
+          test -z "`docker ps|grep sig-test`"
+          if [ $? -ne 0 ]
+          then
+            echo "Looks like container was running when it shouldnt be"
+            docker ps
+            exit 1
+          fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists           
+          docker rm sig-test
+
+      - name: Dump container log
+        if: always()
+        run: |
+          mkdir output-logs
+          docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout-${{ env.PYTHON_VERSION }}.txt
+          docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr-${{ env.PYTHON_VERSION }}.txt
+
+      - name: Store container log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-cdio-basic-tests-output-py${{ env.PYTHON_VERSION }}
+          path: output-logs

Dockerfile

@@ -2,7 +2,10 @@
 
 # @NOTE! I would love to move to 3.11 but it breaks the async handler in changedetectionio/content_fetchers/puppeteer.py
 #        If you know how to fix it, please do! and test it for both 3.10 and 3.11
-FROM python:3.10-slim-bookworm as builder
+
+ARG PYTHON_VERSION=3.11
+
+FROM python:${PYTHON_VERSION}-slim-bookworm AS builder
 
 # See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -23,7 +26,8 @@ WORKDIR /install
 
 COPY requirements.txt /requirements.txt
 
-RUN pip install --target=/dependencies -r /requirements.txt
+# --extra-index-url https://www.piwheels.org/simple  is for cryptography module to be prebuilt (or rustc etc needs to be installed)
+RUN pip install --extra-index-url https://www.piwheels.org/simple  --target=/dependencies -r /requirements.txt
 
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
@@ -32,7 +36,7 @@ RUN pip install --target=/dependencies playwright~=1.41.2 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.10-slim-bookworm
+FROM python:${PYTHON_VERSION}-slim-bookworm
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libxslt1.1 \

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.45.23'
+__version__ = '0.45.25'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/api/api_v1.py

@@ -170,23 +170,33 @@ class WatchSingleHistory(Resource):
             curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
         @apiName Get single snapshot content
         @apiGroup Watch History
+        @apiParam {String} [html]       Optional Set to =1 to return the last HTML (only stores last 2 snapshots, use `latest` as timestamp)
         @apiSuccess (200) {String} OK
         @apiSuccess (404) {String} ERR Not found
         """
         watch = self.datastore.data['watching'].get(uuid)
         if not watch:
-            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+            abort(404, message=f"No watch exists with the UUID of {uuid}")
 
         if not len(watch.history):
-            abort(404, message='Watch found but no history exists for the UUID {}'.format(uuid))
+            abort(404, message=f"Watch found but no history exists for the UUID {uuid}")
 
         if timestamp == 'latest':
             timestamp = list(watch.history.keys())[-1]
 
-        content = watch.get_history_snapshot(timestamp)
+        if request.args.get('html'):
+            content = watch.get_fetched_html(timestamp)
+            if content:
+                response = make_response(content, 200)
+                response.mimetype = "text/html"
+            else:
+                response = make_response("No content found", 404)
+                response.mimetype = "text/plain"
+        else:
+            content = watch.get_history_snapshot(timestamp)
+            response = make_response(content, 200)
+            response.mimetype = "text/plain"
 
-        response = make_response(content, 200)
-        response.mimetype = "text/plain"
         return response
 
 

changedetectionio/blueprint/browser_steps/__init__.py

@@ -187,8 +187,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
             if is_last_step and u:
                 (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data()
-                datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
-                datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
+                watch = datastore.data['watching'].get(uuid)
+                if watch:
+                    watch.save_screenshot(screenshot=screenshot)
+                    watch.save_xpath_data(data=xpath_data)
 
 #        if not this_session.page:
 #            cleanup_playwright_session()

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -255,8 +255,9 @@ class browsersteps_live_ui(steppable_browser_interface):
 
     def get_current_state(self):
         """Return the screenshot and interactive elements mapping, generally always called after action_()"""
-        from pkg_resources import resource_string
-        xpath_element_js = resource_string(__name__, "../../content_fetchers/res/xpath_element_scraper.js").decode('utf-8')
+        import importlib.resources
+        xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
+
         now = time.time()
         self.page.wait_for_timeout(1 * 1000)
 
@@ -287,11 +288,9 @@ class browsersteps_live_ui(steppable_browser_interface):
         :param current_include_filters:
         :return:
         """
-
+        import importlib.resources
         self.page.evaluate("var include_filters=''")
-        from pkg_resources import resource_string
-        # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
-        xpath_element_js = resource_string(__name__, "../../content_fetchers/res/xpath_element_scraper.js").decode('utf-8')
+        xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
         from changedetectionio.content_fetchers import visualselector_xpath_selectors
         xpath_element_js = xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
         xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -63,7 +63,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                             <ul>
                                 <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
                                 {% if jq_support %}
-                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
+                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>. Prefix <code>jqraw:</code> outputs the results as text instead of a JSON list.</li>
                                 {% else %}
                                 <li>jq support not installed</li>
                                 {% endif %}

changedetectionio/content_fetchers/base.py

@@ -28,7 +28,7 @@ def manage_user_agent(headers, current_ua=''):
     :return:
     """
     # Ask it what the user agent is, if its obviously ChromeHeadless, switch it to the default
-    ua_in_custom_headers = next((v for k, v in headers.items() if k.lower() == "user-agent"), None)
+    ua_in_custom_headers = headers.get('User-Agent')
     if ua_in_custom_headers:
         return ua_in_custom_headers
 
@@ -64,10 +64,9 @@ class Fetcher():
     render_extract_delay = 0
 
     def __init__(self):
-        from pkg_resources import resource_string
-        # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
-        self.xpath_element_js = resource_string(__name__, "res/xpath_element_scraper.js").decode('utf-8')
-        self.instock_data_js = resource_string(__name__, "res/stock-not-in-stock.js").decode('utf-8')
+        import importlib.resources
+        self.xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
+        self.instock_data_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('stock-not-in-stock.js').read_text()
 
     @abstractmethod
     def get_error(self):

changedetectionio/content_fetchers/exceptions/__init__.py

@@ -87,11 +87,12 @@ class ScreenshotUnavailable(Exception):
 
 
 class ReplyWithContentButNoText(Exception):
-    def __init__(self, status_code, url, screenshot=None, has_filters=False, html_content=''):
+    def __init__(self, status_code, url, screenshot=None, has_filters=False, html_content='', xpath_data=None):
         # Set this so we can use it in other parts of the app
         self.status_code = status_code
         self.url = url
         self.screenshot = screenshot
         self.has_filters = has_filters
         self.html_content = html_content
+        self.xpath_data = xpath_data
         return

changedetectionio/content_fetchers/puppeteer.py

@@ -115,12 +115,11 @@ class fetcher(Fetcher):
 
         # This user agent is similar to what was used when tweaking the evasions in inject_evasions_into_page(..)
         user_agent = None
-        if request_headers:
-            user_agent = next((value for key, value in request_headers.items() if key.lower().strip() == 'user-agent'), None)
-            if user_agent:
-                await self.page.setUserAgent(user_agent)
-                # Remove it so it's not sent again with headers after
-                [request_headers.pop(key) for key in list(request_headers) if key.lower().strip() == 'user-agent'.lower().strip()]
+        if request_headers and request_headers.get('User-Agent'):
+            # Request_headers should now be CaaseInsensitiveDict
+            # Remove it so it's not sent again with headers after
+            user_agent = request_headers.pop('User-Agent').strip()
+            await self.page.setUserAgent(user_agent)
 
         if not user_agent:
             # Attempt to strip 'HeadlessChrome' etc

changedetectionio/content_fetchers/res/__init__.py

@@ -0,0 +1 @@
+# resources for browser injection/scraping

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -30,14 +30,21 @@ function isItemInStock() {
         'dieser artikel ist bald wieder verfügbar',
         'dostępne wkrótce',
         'en rupture de stock',
-        'ist derzeit nicht auf lager',
+        'isn\'t in stock right now',
+        'isnt in stock right now',
+        'isn’t in stock right now',
         'item is no longer available',
         'let me know when it\'s available',
+        'mail me when available',
         'message if back in stock',
         'nachricht bei',
         'nicht auf lager',
+        'nicht lagernd',
         'nicht lieferbar',
+        'nicht verfügbar',
+        'nicht vorrätig',
         'nicht zur verfügung',
+        'nie znaleziono produktów',
         'niet beschikbaar',
         'niet leverbaar',
         'niet op voorraad',
@@ -48,6 +55,7 @@ function isItemInStock() {
         'not currently available',
         'not in stock',
         'notify me when available',
+        'notify me',
         'notify when available',
         'não estamos a aceitar encomendas',
         'out of stock',
@@ -62,12 +70,16 @@ function isItemInStock() {
         'this item is currently unavailable',
         'tickets unavailable',
         'tijdelijk uitverkocht',
+        'unavailable nearby',
         'unavailable tickets',
+        'vergriffen',
+        'vorbestellen',
         'vorbestellung ist bald möglich',
         'we couldn\'t find any products that match',
         'we do not currently have an estimate of when this product will be back in stock.',
         'we don\'t know when or if this item will be back in stock.',
         'we were not able to find a match',
+        'when this arrives in stock',
         'zur zeit nicht an lager',
         '品切れ',
         '已售',

changedetectionio/content_fetchers/res/xpath_element_scraper.js

@@ -182,6 +182,7 @@ visibleElementsArray.forEach(function (element) {
 // Inject the current one set in the include_filters, which may be a CSS rule
 // used for displaying the current one in VisualSelector, where its not one we generated.
 if (include_filters.length) {
+    let results;
     // Foreach filter, go and find it on the page and add it to the results so we can visualise it again
     for (const f of include_filters) {
         bbox = false;
@@ -197,10 +198,15 @@ if (include_filters.length) {
             if (f.startsWith('/') || f.startsWith('xpath')) {
                 var qry_f = f.replace(/xpath(:|\d:)/, '')
                 console.log("[xpath] Scanning for included filter " + qry_f)
-                q = document.evaluate(qry_f, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+                let xpathResult = document.evaluate(qry_f, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+                results = [];
+                for (let i = 0; i < xpathResult.snapshotLength; i++) {
+                    results.push(xpathResult.snapshotItem(i));
+                }
             } else {
                 console.log("[css] Scanning for included filter " + f)
-                q = document.querySelector(f);
+                console.log("[css] Scanning for included filter " + f);
+                results = document.querySelectorAll(f);
             }
         } catch (e) {
             // Maybe catch DOMException and alert?
@@ -208,44 +214,45 @@ if (include_filters.length) {
             console.log(e);
         }
 
-        if (q) {
-            // Try to resolve //something/text() back to its /something so we can atleast get the bounding box
-            try {
-                if (typeof q.nodeName == 'string' && q.nodeName === '#text') {
-                    q = q.parentElement
-                }
-            } catch (e) {
-                console.log(e)
-                console.log("xpath_element_scraper: #text resolver")
-            }
+        if (results.length) {
 
-            // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
-            if (typeof q.getBoundingClientRect == 'function') {
-                bbox = q.getBoundingClientRect();
-                console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
-            } else {
+            // Iterate over the results
+            results.forEach(node => {
+                // Try to resolve //something/text() back to its /something so we can atleast get the bounding box
                 try {
-                    // Try and see we can find its ownerElement
-                    bbox = q.ownerElement.getBoundingClientRect();
-                    console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
+                    if (typeof node.nodeName == 'string' && node.nodeName === '#text') {
+                        node = node.parentElement
+                    }
                 } catch (e) {
                     console.log(e)
-                    console.log("xpath_element_scraper: error looking up q.ownerElement")
+                    console.log("xpath_element_scraper: #text resolver")
                 }
-            }
-        }
 
-        if (!q) {
-            console.log("xpath_element_scraper: filter element " + f + " was not found");
-        }
+                // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
+                if (typeof node.getBoundingClientRect == 'function') {
+                    bbox = node.getBoundingClientRect();
+                    console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
+                } else {
+                    try {
+                        // Try and see we can find its ownerElement
+                        bbox = node.ownerElement.getBoundingClientRect();
+                        console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
+                    } catch (e) {
+                        console.log(e)
+                        console.log("xpath_element_scraper: error looking up q.ownerElement")
+                    }
+                }
 
-        if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
-            size_pos.push({
-                xpath: f,
-                width: parseInt(bbox['width']),
-                height: parseInt(bbox['height']),
-                left: parseInt(bbox['left']),
-                top: parseInt(bbox['top']) + scroll_y
+                if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
+                    size_pos.push({
+                        xpath: f,
+                        width: parseInt(bbox['width']),
+                        height: parseInt(bbox['height']),
+                        left: parseInt(bbox['left']),
+                        top: parseInt(bbox['top']) + scroll_y,
+                        highlight_as_custom_filter: true
+                    });
+                }
             });
         }
     }

changedetectionio/diff.py

@@ -1,62 +1,97 @@
-# used for the notifications, the front-end is using a JS library
-
 import difflib
+from typing import List, Iterator, Union
 
+def same_slicer(lst: List[str], start: int, end: int) -> List[str]:
+    """Return a slice of the list, or a single element if start == end."""
+    return lst[start:end] if start != end else [lst[start]]
 
-def same_slicer(l, a, b):
-    if a == b:
-        return [l[a]]
-    else:
-        return l[a:b]
-
-# like .compare but a little different output
-def customSequenceMatcher(before, after, include_equal=False, include_removed=True, include_added=True, include_replaced=True, include_change_type_prefix=True):
-    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
-
-    # @todo Line-by-line mode instead of buncghed, including `after` that is not in `before` (maybe unset?)
+def customSequenceMatcher(
+    before: List[str],
+    after: List[str],
+    include_equal: bool = False,
+    include_removed: bool = True,
+    include_added: bool = True,
+    include_replaced: bool = True,
+    include_change_type_prefix: bool = True
+) -> Iterator[List[str]]:
+    """
+    Compare two sequences and yield differences based on specified parameters.
+    
+    Args:
+        before (List[str]): Original sequence
+        after (List[str]): Modified sequence
+        include_equal (bool): Include unchanged parts
+        include_removed (bool): Include removed parts
+        include_added (bool): Include added parts
+        include_replaced (bool): Include replaced parts
+        include_change_type_prefix (bool): Add prefixes to indicate change types
+    
+    Yields:
+        List[str]: Differences between sequences
+    """
+    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \t", a=before, b=after)
+    
     for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
         if include_equal and tag == 'equal':
-            g = before[alo:ahi]
-            yield g
+            yield before[alo:ahi]
         elif include_removed and tag == 'delete':
-            row_prefix = "(removed) " if include_change_type_prefix else ''
-            g = [ row_prefix + i for i in same_slicer(before, alo, ahi)]
-            yield g
+            prefix = "(removed) " if include_change_type_prefix else ''
+            yield [f"{prefix}{line}" for line in same_slicer(before, alo, ahi)]
         elif include_replaced and tag == 'replace':
-            row_prefix = "(changed) " if include_change_type_prefix else ''
-            g = [row_prefix + i for i in same_slicer(before, alo, ahi)]
-            row_prefix = "(into) " if include_change_type_prefix else ''
-            g += [row_prefix + i for i in same_slicer(after, blo, bhi)]
-            yield g
+            prefix_changed = "(changed) " if include_change_type_prefix else ''
+            prefix_into = "(into) " if include_change_type_prefix else ''
+            yield [f"{prefix_changed}{line}" for line in same_slicer(before, alo, ahi)] + \
+                  [f"{prefix_into}{line}" for line in same_slicer(after, blo, bhi)]
         elif include_added and tag == 'insert':
-            row_prefix = "(added) " if include_change_type_prefix else ''
-            g = [row_prefix + i for i in same_slicer(after, blo, bhi)]
-            yield g
+            prefix = "(added) " if include_change_type_prefix else ''
+            yield [f"{prefix}{line}" for line in same_slicer(after, blo, bhi)]
 
-# only_differences - only return info about the differences, no context
-# line_feed_sep could be "<br>" or "<li>" or "\n" etc
-def render_diff(previous_version_file_contents, newest_version_file_contents, include_equal=False, include_removed=True, include_added=True, include_replaced=True, line_feed_sep="\n", include_change_type_prefix=True, patch_format=False):
-
-    newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
-
-    if previous_version_file_contents:
-        previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
-    else:
-        previous_version_file_contents = ""
+def render_diff(
+    previous_version_file_contents: str,
+    newest_version_file_contents: str,
+    include_equal: bool = False,
+    include_removed: bool = True,
+    include_added: bool = True,
+    include_replaced: bool = True,
+    line_feed_sep: str = "\n",
+    include_change_type_prefix: bool = True,
+    patch_format: bool = False
+) -> str:
+    """
+    Render the difference between two file contents.
+    
+    Args:
+        previous_version_file_contents (str): Original file contents
+        newest_version_file_contents (str): Modified file contents
+        include_equal (bool): Include unchanged parts
+        include_removed (bool): Include removed parts
+        include_added (bool): Include added parts
+        include_replaced (bool): Include replaced parts
+        line_feed_sep (str): Separator for lines in output
+        include_change_type_prefix (bool): Add prefixes to indicate change types
+        patch_format (bool): Use patch format for output
+    
+    Returns:
+        str: Rendered difference
+    """
+    newest_lines = [line.rstrip() for line in newest_version_file_contents.splitlines()]
+    previous_lines = [line.rstrip() for line in previous_version_file_contents.splitlines()] if previous_version_file_contents else []
 
     if patch_format:
-        patch = difflib.unified_diff(previous_version_file_contents, newest_version_file_contents)
+        patch = difflib.unified_diff(previous_lines, newest_lines)
         return line_feed_sep.join(patch)
 
-    rendered_diff = customSequenceMatcher(before=previous_version_file_contents,
-                                          after=newest_version_file_contents,
-                                          include_equal=include_equal,
-                                          include_removed=include_removed,
-                                          include_added=include_added,
-                                          include_replaced=include_replaced,
-                                          include_change_type_prefix=include_change_type_prefix)
+    rendered_diff = customSequenceMatcher(
+        before=previous_lines,
+        after=newest_lines,
+        include_equal=include_equal,
+        include_removed=include_removed,
+        include_added=include_added,
+        include_replaced=include_replaced,
+        include_change_type_prefix=include_change_type_prefix
+    )
 
-    # Recursively join lists
-    f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
-    p= f(rendered_diff)
-    return p
+    def flatten(lst: List[Union[str, List[str]]]) -> str:
+        return line_feed_sep.join(flatten(x) if isinstance(x, list) else x for x in lst)
+
+    return flatten(rendered_diff)

changedetectionio/flask_app.py

@@ -339,7 +339,7 @@ def changedetection_app(config=None, datastore_o=None):
         # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
         for uuid, watch in datastore.data['watching'].items():
             # @todo tag notification_muted skip also (improve Watch model)
-            if watch.get('notification_muted'):
+            if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'):
                 continue
             if limit_tag and not limit_tag in watch['tags']:
                 continue
@@ -472,7 +472,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  # Don't link to hosting when we're on the hosting environment
                                  active_tag=active_tag,
                                  active_tag_uuid=active_tag_uuid,
-                                 app_rss_token=datastore.data['settings']['application']['rss_access_token'],
+                                 app_rss_token=datastore.data['settings']['application'].get('rss_access_token'),
                                  datastore=datastore,
                                  errored_count=errored_count,
                                  form=form,
@@ -679,7 +679,10 @@ def changedetection_app(config=None, datastore_o=None):
 
         if request.method == 'POST' and form.validate():
 
-            extra_update_obj = {}
+            extra_update_obj = {
+                'consecutive_filter_failures': 0,
+                'last_error' : False
+            }
 
             if request.args.get('unpause_on_save'):
                 extra_update_obj['paused'] = False
@@ -718,7 +721,7 @@ def changedetection_app(config=None, datastore_o=None):
             datastore.data['watching'][uuid].update(extra_update_obj)
 
             if request.args.get('unpause_on_save'):
-                flash("Updated watch - unpaused!.")
+                flash("Updated watch - unpaused!")
             else:
                 flash("Updated watch.")
 

changedetectionio/forms.py

@@ -572,6 +572,8 @@ class globalSettingsApplicationForm(commonSettingsForm):
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
     shared_diff_access = BooleanField('Allow access to view diff page when password is enabled', default=False, validators=[validators.Optional()])
+    rss_hide_muted_watches = BooleanField('Hide muted watches from RSS feed', default=True,
+                                      validators=[validators.Optional()])
     filter_failure_notification_threshold_attempts = IntegerField('Number of times the filter can be missing before sending a notification',
                                                                   render_kw={"style": "width: 5em;"},
                                                                   validators=[validators.NumberRange(min=0,

changedetectionio/html_tools.py

@@ -3,8 +3,6 @@ from bs4 import BeautifulSoup
 from inscriptis import get_text
 from jsonpath_ng.ext import parse
 from typing import List
-from inscriptis.css_profiles import CSS_PROFILES, HtmlElement
-from inscriptis.html_properties import Display
 from inscriptis.model.config import ParserConfig
 from xml.sax.saxutils import escape as xml_escape
 import json
@@ -196,12 +194,12 @@ def extract_element(find='title', html_content=''):
 
 #
 def _parse_json(json_data, json_filter):
-    if 'json:' in json_filter:
+    if json_filter.startswith("json:"):
         jsonpath_expression = parse(json_filter.replace('json:', ''))
         match = jsonpath_expression.find(json_data)
         return _get_stripped_text_from_json_match(match)
 
-    if 'jq:' in json_filter:
+    if json_filter.startswith("jq:") or json_filter.startswith("jqraw:"):
 
         try:
             import jq
@@ -209,10 +207,15 @@ def _parse_json(json_data, json_filter):
             # `jq` requires full compilation in windows and so isn't generally available
             raise Exception("jq not support not found")
 
-        jq_expression = jq.compile(json_filter.replace('jq:', ''))
-        match = jq_expression.input(json_data).all()
+        if json_filter.startswith("jq:"):
+            jq_expression = jq.compile(json_filter.removeprefix("jq:"))
+            match = jq_expression.input(json_data).all()
+            return _get_stripped_text_from_json_match(match)
 
-        return _get_stripped_text_from_json_match(match)
+        if json_filter.startswith("jqraw:"):
+            jq_expression = jq.compile(json_filter.removeprefix("jqraw:"))
+            match = jq_expression.input(json_data).all()
+            return '\n'.join(str(item) for item in match)
 
 def _get_stripped_text_from_json_match(match):
     s = []

changedetectionio/model/App.py

@@ -5,6 +5,7 @@ from changedetectionio.notification import (
     default_notification_title,
 )
 
+# Equal to or greater than this number of FilterNotFoundInResponse exceptions will trigger a filter-not-found notification
 _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT = 6
 DEFAULT_SETTINGS_HEADERS_USERAGENT='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'
 
@@ -46,6 +47,8 @@ class model(dict):
                     'pager_size': 50,
                     'password': False,
                     'render_anchor_tag_content': False,
+                    'rss_access_token': None,
+                    'rss_hide_muted_watches': True,
                     'schema_version' : 0,
                     'shared_diff_access': False,
                     'webdriver_delay': None , # Extra delay in seconds before extracting text

changedetectionio/model/Watch.py

@@ -238,6 +238,8 @@ class model(dict):
 
         if len(tmp_history):
             self.__newest_history_key = list(tmp_history.keys())[-1]
+        else:
+            self.__newest_history_key = None
 
         self.__history_n = len(tmp_history)
 
@@ -328,12 +330,9 @@ class model(dict):
     def save_history_text(self, contents, timestamp, snapshot_id):
         import brotli
 
-        self.ensure_data_dir_exists()
+        logger.trace(f"{self.get('uuid')} - Updating history.txt with timestamp {timestamp}")
 
-        # Small hack so that we sleep just enough to allow 1 second  between history snapshots
-        # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
-        if self.__newest_history_key and int(timestamp) == int(self.__newest_history_key):
-            time.sleep(timestamp - self.__newest_history_key)
+        self.ensure_data_dir_exists()
 
         threshold = int(os.getenv('SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD', 1024))
         skip_brotli = strtobool(os.getenv('DISABLE_BROTLI_TEXT_SNAPSHOT', 'False'))
@@ -526,8 +525,42 @@ class model(dict):
         # None is set
         return False
 
+    def save_error_text(self, contents):
+        self.ensure_data_dir_exists()
+        target_path = os.path.join(self.watch_data_dir, "last-error.txt")
+        with open(target_path, 'w') as f:
+            f.write(contents)
 
-    def get_last_fetched_before_filters(self):
+    def save_xpath_data(self, data, as_error=False):
+        import json
+
+        if as_error:
+            target_path = os.path.join(self.watch_data_dir, "elements-error.json")
+        else:
+            target_path = os.path.join(self.watch_data_dir, "elements.json")
+
+        self.ensure_data_dir_exists()
+
+        with open(target_path, 'w') as f:
+            f.write(json.dumps(data))
+            f.close()
+
+    # Save as PNG, PNG is larger but better for doing visual diff in the future
+    def save_screenshot(self, screenshot: bytes, as_error=False):
+
+        if as_error:
+            target_path = os.path.join(self.watch_data_dir, "last-error-screenshot.png")
+        else:
+            target_path = os.path.join(self.watch_data_dir, "last-screenshot.png")
+
+        self.ensure_data_dir_exists()
+
+        with open(target_path, 'wb') as f:
+            f.write(screenshot)
+            f.close()
+
+
+    def get_last_fetched_text_before_filters(self):
         import brotli
         filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
 
@@ -542,12 +575,56 @@ class model(dict):
         with open(filepath, 'rb') as f:
             return(brotli.decompress(f.read()).decode('utf-8'))
 
-    def save_last_fetched_before_filters(self, contents):
+    def save_last_text_fetched_before_filters(self, contents):
         import brotli
         filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
         with open(filepath, 'wb') as f:
             f.write(brotli.compress(contents, mode=brotli.MODE_TEXT))
 
+    def save_last_fetched_html(self, timestamp, contents):
+        import brotli
+
+        self.ensure_data_dir_exists()
+        snapshot_fname = f"{timestamp}.html.br"
+        filepath = os.path.join(self.watch_data_dir, snapshot_fname)
+
+        with open(filepath, 'wb') as f:
+            contents = contents.encode('utf-8') if isinstance(contents, str) else contents
+            try:
+                f.write(brotli.compress(contents))
+            except Exception as e:
+                logger.warning(f"{self.get('uuid')} - Unable to compress snapshot, saving as raw data to {filepath}")
+                logger.warning(e)
+                f.write(contents)
+
+        self._prune_last_fetched_html_snapshots()
+
+    def get_fetched_html(self, timestamp):
+        import brotli
+
+        snapshot_fname = f"{timestamp}.html.br"
+        filepath = os.path.join(self.watch_data_dir, snapshot_fname)
+        if os.path.isfile(filepath):
+            with open(filepath, 'rb') as f:
+                return (brotli.decompress(f.read()).decode('utf-8'))
+
+        return False
+
+
+    def _prune_last_fetched_html_snapshots(self):
+
+        dates = list(self.history.keys())
+        dates.reverse()
+
+        for index, timestamp in enumerate(dates):
+            snapshot_fname = f"{timestamp}.html.br"
+            filepath = os.path.join(self.watch_data_dir, snapshot_fname)
+
+            # Keep only the first 2
+            if index > 1 and os.path.isfile(filepath):
+                os.remove(filepath)
+
+
     @property
     def get_browsersteps_available_screenshots(self):
         "For knowing which screenshots are available to show the user in BrowserSteps UI"

changedetectionio/processors/__init__.py

@@ -1,10 +1,11 @@
 from abc import abstractmethod
-import os
-import hashlib
-import re
-from copy import deepcopy
 from changedetectionio.strtobool import strtobool
+from changedetectionio.model import Watch
+from copy import deepcopy
 from loguru import logger
+import hashlib
+import os
+import re
 
 class difference_detection_processor():
 
@@ -21,7 +22,7 @@ class difference_detection_processor():
         self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
 
     def call_browser(self):
-
+        from requests.structures import CaseInsensitiveDict
         # Protect against file:// access
         if re.search(r'^file://', self.watch.get('url', '').strip(), re.IGNORECASE):
             if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
@@ -93,14 +94,16 @@ class difference_detection_processor():
             self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid'))
 
         # Tweak the base config with the per-watch ones
-        request_headers = self.watch.get('headers', [])
-        request_headers.update(self.datastore.get_all_base_headers())
-        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=self.watch.get('uuid')))
+        request_headers = CaseInsensitiveDict()
 
         ua = self.datastore.data['settings']['requests'].get('default_ua')
         if ua and ua.get(prefer_fetch_backend):
             request_headers.update({'User-Agent': ua.get(prefer_fetch_backend)})
 
+        request_headers.update(self.watch.get('headers', {}))
+        request_headers.update(self.datastore.get_all_base_headers())
+        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=self.watch.get('uuid')))
+
         # https://github.com/psf/requests/issues/4525
         # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
         # do this by accident.
@@ -136,7 +139,7 @@ class difference_detection_processor():
         # After init, call run_changedetection() which will do the actual change-detection
 
     @abstractmethod
-    def run_changedetection(self, uuid, skip_when_checksum_same=True):
+    def run_changedetection(self, watch: Watch, skip_when_checksum_same=True):
         update_obj = {'last_notification_error': False, 'last_error': False}
         some_data = 'xxxxx'
         update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()

changedetectionio/processors/restock_diff.py

@@ -1,6 +1,5 @@
 
 from . import difference_detection_processor
-from copy import deepcopy
 from loguru import logger
 import hashlib
 import urllib3
@@ -20,10 +19,7 @@ class perform_site_check(difference_detection_processor):
     screenshot = None
     xpath_data = None
 
-    def run_changedetection(self, uuid, skip_when_checksum_same=True):
-
-        # DeepCopy so we can be sure we don't accidently change anything by reference
-        watch = deepcopy(self.datastore.data['watching'].get(uuid))
+    def run_changedetection(self, watch, skip_when_checksum_same=True):
 
         if not watch:
             raise Exception("Watch no longer exists.")
@@ -44,13 +40,13 @@ class perform_site_check(difference_detection_processor):
             fetched_md5 = hashlib.md5(self.fetcher.instock_data.encode('utf-8')).hexdigest()
             # 'Possibly in stock' comes from stock-not-in-stock.js when no string found above the fold.
             update_obj["in_stock"] = True if self.fetcher.instock_data == 'Possibly in stock' else False
-            logger.debug(f"Watch UUID {uuid} restock check returned '{self.fetcher.instock_data}' from JS scraper.")
+            logger.debug(f"Watch UUID {watch.get('uuid')} restock check returned '{self.fetcher.instock_data}' from JS scraper.")
         else:
             raise UnableToExtractRestockData(status_code=self.fetcher.status_code)
 
         # The main thing that all this at the moment comes down to :)
         changed_detected = False
-        logger.debug(f"Watch UUID {uuid} restock check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
+        logger.debug(f"Watch UUID {watch.get('uuid')} restock check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
 
         if watch.get('previous_md5') and watch.get('previous_md5') != fetched_md5:
             # Yes if we only care about it going to instock, AND we are in stock

changedetectionio/processors/text_json_diff.py

@@ -10,18 +10,18 @@ from . import difference_detection_processor
 from ..html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text
 from changedetectionio import html_tools, content_fetchers
 from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
-import changedetectionio.content_fetchers
-from copy import deepcopy
 from loguru import logger
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 name = 'Webpage Text/HTML, JSON and PDF changes'
 description = 'Detects all text changes where possible'
-json_filter_prefixes = ['json:', 'jq:']
+json_filter_prefixes = ['json:', 'jq:', 'jqraw:']
 
 class FilterNotFoundInResponse(ValueError):
-    def __init__(self, msg):
+    def __init__(self, msg, screenshot=None, xpath_data=None):
+        self.screenshot = screenshot
+        self.xpath_data = xpath_data
         ValueError.__init__(self, msg)
 
 
@@ -34,14 +34,12 @@ class PDFToHTMLToolNotFound(ValueError):
 # (set_proxy_from_list)
 class perform_site_check(difference_detection_processor):
 
-    def run_changedetection(self, uuid, skip_when_checksum_same=True):
+    def run_changedetection(self, watch, skip_when_checksum_same=True):
         changed_detected = False
         html_content = ""
         screenshot = False  # as bytes
         stripped_text_from_html = ""
 
-        # DeepCopy so we can be sure we don't accidently change anything by reference
-        watch = deepcopy(self.datastore.data['watching'].get(uuid))
         if not watch:
             raise Exception("Watch no longer exists.")
 
@@ -116,12 +114,12 @@ class perform_site_check(difference_detection_processor):
         # Better would be if Watch.model could access the global data also
         # and then use getattr https://docs.python.org/3/reference/datamodel.html#object.__getitem__
         # https://realpython.com/inherit-python-dict/ instead of doing it procedurely
-        include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='include_filters')
+        include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='include_filters')
 
         # 1845 - remove duplicated filters in both group and watch include filter
         include_filters_rule = list(dict.fromkeys(watch.get('include_filters', []) + include_filters_from_tags))
 
-        subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='subtractive_selectors'),
+        subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='subtractive_selectors'),
                                  *watch.get("subtractive_selectors", []),
                                  *self.datastore.data["settings"]["application"].get("global_subtractive_selectors", [])
                                  ]
@@ -188,7 +186,7 @@ class perform_site_check(difference_detection_processor):
                                                                        append_pretty_line_formatting=not watch.is_source_type_url)
 
                     if not html_content.strip():
-                        raise FilterNotFoundInResponse(include_filters_rule)
+                        raise FilterNotFoundInResponse(msg=include_filters_rule, screenshot=self.fetcher.screenshot, xpath_data=self.fetcher.xpath_data)
 
                 if has_subtractive_selectors:
                     html_content = html_tools.element_removal(subtractive_selectors, html_content)
@@ -222,7 +220,7 @@ class perform_site_check(difference_detection_processor):
             from .. import diff
             # needs to not include (added) etc or it may get used twice
             # Replace the processed text with the preferred result
-            rendered_diff = diff.render_diff(previous_version_file_contents=watch.get_last_fetched_before_filters(),
+            rendered_diff = diff.render_diff(previous_version_file_contents=watch.get_last_fetched_text_before_filters(),
                                              newest_version_file_contents=stripped_text_from_html,
                                              include_equal=False,  # not the same lines
                                              include_added=watch.get('filter_text_added', True),
@@ -231,7 +229,7 @@ class perform_site_check(difference_detection_processor):
                                              line_feed_sep="\n",
                                              include_change_type_prefix=False)
 
-            watch.save_last_fetched_before_filters(text_content_before_ignored_filter)
+            watch.save_last_text_fetched_before_filters(text_content_before_ignored_filter)
 
             if not rendered_diff and stripped_text_from_html:
                 # We had some content, but no differences were found
@@ -246,9 +244,10 @@ class perform_site_check(difference_detection_processor):
         if not is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0:
             raise content_fetchers.exceptions.ReplyWithContentButNoText(url=url,
                                                             status_code=self.fetcher.get_last_status_code(),
-                                                            screenshot=screenshot,
+                                                            screenshot=self.fetcher.screenshot,
                                                             has_filters=has_filter_rule,
-                                                            html_content=html_content
+                                                            html_content=html_content,
+                                                            xpath_data=self.fetcher.xpath_data
                                                             )
 
         # We rely on the actual text in the html output.. many sites have random script vars etc,
@@ -344,17 +343,17 @@ class perform_site_check(difference_detection_processor):
                 if not watch['title'] or not len(watch['title']):
                     update_obj['title'] = html_tools.extract_element(find='title', html_content=self.fetcher.content)
 
-        logger.debug(f"Watch UUID {uuid} content check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
+        logger.debug(f"Watch UUID {watch.get('uuid')} content check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
 
         if changed_detected:
             if watch.get('check_unique_lines', False):
                 has_unique_lines = watch.lines_contain_something_unique_compared_to_history(lines=stripped_text_from_html.splitlines())
                 # One or more lines? unsure?
                 if not has_unique_lines:
-                    logger.debug(f"check_unique_lines: UUID {uuid} didnt have anything new setting change_detected=False")
+                    logger.debug(f"check_unique_lines: UUID {watch.get('uuid')} didnt have anything new setting change_detected=False")
                     changed_detected = False
                 else:
-                    logger.debug(f"check_unique_lines: UUID {uuid} had unique content")
+                    logger.debug(f"check_unique_lines: UUID {watch.get('uuid')} had unique content")
 
         # Always record the new checksum
         update_obj["previous_md5"] = fetched_md5

changedetectionio/static/js/browser-steps.js

@@ -1,14 +1,5 @@
 $(document).ready(function () {
 
-    // duplicate
-    var csrftoken = $('input[name=csrf_token]').val();
-    $.ajaxSetup({
-        beforeSend: function (xhr, settings) {
-            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
-                xhr.setRequestHeader("X-CSRFToken", csrftoken)
-            }
-        }
-    })
     var browsersteps_session_id;
     var browser_interface_seconds_remaining = 0;
     var apply_buttons_disabled = false;

changedetectionio/static/js/csrf.js

@@ -0,0 +1,10 @@
+$(document).ready(function () {
+    $.ajaxSetup({
+        beforeSend: function (xhr, settings) {
+            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
+                xhr.setRequestHeader("X-CSRFToken", csrftoken)
+            }
+        }
+    })
+});
+

changedetectionio/static/js/diff-overview.js

@@ -1,13 +1,4 @@
 $(document).ready(function () {
-    var csrftoken = $('input[name=csrf_token]').val();
-    $.ajaxSetup({
-        beforeSend: function (xhr, settings) {
-            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
-                xhr.setRequestHeader("X-CSRFToken", csrftoken)
-            }
-        }
-    })
-
     $('.needs-localtime').each(function () {
         for (var option of this.options) {
             var dateObject = new Date(option.value * 1000);
@@ -48,6 +39,12 @@ $(document).ready(function () {
       $("#highlightSnippet").remove();
     }
 
+    // Listen for Escape key press
+    window.addEventListener('keydown', function (e) {
+        if (e.key === 'Escape') {
+            clean();
+        }
+    }, false);
 
     function dragTextHandler(event) {
         console.log('mouseupped');

changedetectionio/static/js/notifications.js

@@ -13,16 +13,6 @@ $(document).ready(function() {
   $('#send-test-notification').click(function (e) {
     e.preventDefault();
 
-    // this can be global
-    var csrftoken = $('input[name=csrf_token]').val();
-    $.ajaxSetup({
-        beforeSend: function(xhr, settings) {
-            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
-                xhr.setRequestHeader("X-CSRFToken", csrftoken)
-            }
-        }
-    })
-
     data = {
       notification_body: $('#notification_body').val(),
       notification_format: $('#notification_format').val(),

changedetectionio/static/js/visual-selector.js

@@ -2,250 +2,258 @@
 // All rights reserved.
 // yes - this is really a hack, if you are a front-ender and want to help, please get in touch!
 
-$(document).ready(function () {
+let runInClearMode = false;
 
-    var current_selected_i;
-    var state_clicked = false;
+$(document).ready(() => {
+    let currentSelections = [];
+    let currentSelection = null;
+    let appendToList = false;
+    let c, xctx, ctx;
+    let xScale = 1, yScale = 1;
+    let selectorImage, selectorImageRect, selectorData;
 
-    var c;
 
-    // greyed out fill context
-    var xctx;
-    // redline highlight context
-    var ctx;
+    // Global jQuery selectors with "Elem" appended
+    const $selectorCanvasElem = $('#selector-canvas');
+    const $includeFiltersElem = $("#include_filters");
+    const $selectorBackgroundElem = $("img#selector-background");
+    const $selectorCurrentXpathElem = $("#selector-current-xpath span");
+    const $fetchingUpdateNoticeElem = $('.fetching-update-notice');
+    const $selectorWrapperElem = $("#selector-wrapper");
 
-    var current_default_xpath = [];
-    var x_scale = 1;
-    var y_scale = 1;
-    var selector_image;
-    var selector_image_rect;
-    var selector_data;
+    // Color constants
+    const FILL_STYLE_HIGHLIGHT = 'rgba(205,0,0,0.35)';
+    const FILL_STYLE_GREYED_OUT = 'rgba(205,205,205,0.95)';
+    const STROKE_STYLE_HIGHLIGHT = 'rgba(255,0,0, 0.9)';
+    const FILL_STYLE_REDLINE = 'rgba(255,0,0, 0.1)';
+    const STROKE_STYLE_REDLINE = 'rgba(225,0,0,0.9)';
 
-    $('#visualselector-tab').click(function () {
-        $("img#selector-background").off('load');
-        state_clicked = false;
-        current_selected_i = false;
-        bootstrap_visualselector();
+    $('#visualselector-tab').click(() => {
+        $selectorBackgroundElem.off('load');
+        currentSelections = [];
+        bootstrapVisualSelector();
     });
 
-    $(document).on('keydown', function (event) {
-        if ($("img#selector-background").is(":visible")) {
-            if (event.key == "Escape") {
-                state_clicked = false;
-                ctx.clearRect(0, 0, c.width, c.height);
+    function clearReset() {
+        ctx.clearRect(0, 0, c.width, c.height);
+
+        if ($includeFiltersElem.val().length) {
+            alert("Existing filters under the 'Filters & Triggers' tab were cleared.");
+        }
+        $includeFiltersElem.val('');
+
+        currentSelections = [];
+
+        // Means we ignore the xpaths from the scraper marked as sel.highlight_as_custom_filter (it matched a previous selector)
+        runInClearMode = true;
+
+        highlightCurrentSelected();
+    }
+
+    function splitToList(v) {
+        return v.split('\n').map(line => line.trim()).filter(line => line.length > 0);
+    }
+
+    function sortScrapedElementsBySize() {
+        // Sort the currentSelections array by area (width * height) in descending order
+        selectorData['size_pos'].sort((a, b) => {
+            const areaA = a.width * a.height;
+            const areaB = b.width * b.height;
+            return areaB - areaA;
+        });
+    }
+
+    $(document).on('keydown keyup', (event) => {
+        if (event.code === 'ShiftLeft' || event.code === 'ShiftRight') {
+            appendToList = event.type === 'keydown';
+        }
+
+        if (event.type === 'keydown') {
+            if ($selectorBackgroundElem.is(":visible") && event.key === "Escape") {
+                clearReset();
             }
         }
     });
 
-    // For when the page loads
-    if (!window.location.hash || window.location.hash != '#visualselector') {
-        $("img#selector-background").attr('src', '');
+    $('#clear-selector').on('click', () => {
+        clearReset();
+    });
+    // So if they start switching between visualSelector and manual filters, stop it from rendering old filters
+    $('li.tab a').on('click', () => {
+        runInClearMode = true;
+    });
+
+    if (!window.location.hash || window.location.hash !== '#visualselector') {
+        $selectorBackgroundElem.attr('src', '');
         return;
     }
 
-    // Handle clearing button/link
-    $('#clear-selector').on('click', function (event) {
-        if (!state_clicked) {
-            alert('Oops, Nothing selected!');
-        }
-        state_clicked = false;
-        ctx.clearRect(0, 0, c.width, c.height);
-        xctx.clearRect(0, 0, c.width, c.height);
-        $("#include_filters").val('');
-    });
+    bootstrapVisualSelector();
 
-
-    bootstrap_visualselector();
-
-
-    function bootstrap_visualselector() {
-        if (1) {
-            // bootstrap it, this will trigger everything else
-            $("img#selector-background").on("error", function () {
-                $('.fetching-update-notice').html("<strong>Ooops!</strong> The VisualSelector tool needs atleast one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.");
-                $('.fetching-update-notice').css('color','#bb0000');
-                $('#selector-current-xpath').hide();
-                $('#clear-selector').hide();
-            }).bind('load', function () {
+    function bootstrapVisualSelector() {
+        $selectorBackgroundElem
+            .on("error", () => {
+                $fetchingUpdateNoticeElem.html("<strong>Ooops!</strong> The VisualSelector tool needs at least one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.")
+                    .css('color', '#bb0000');
+                $('#selector-current-xpath, #clear-selector').hide();
+            })
+            .on('load', () => {
                 console.log("Loaded background...");
                 c = document.getElementById("selector-canvas");
-                // greyed out fill context
                 xctx = c.getContext("2d");
-                // redline highlight context
                 ctx = c.getContext("2d");
-                if ($("#include_filters").val().trim().length) {
-                    current_default_xpath = $("#include_filters").val().split(/\r?\n/g);
-                } else {
-                    current_default_xpath = [];
-                }
-                fetch_data();
-                $('#selector-canvas').off("mousemove mousedown");
-                // screenshot_url defined in the edit.html template
-            }).attr("src", screenshot_url);
-        }
-        // Tell visualSelector that the image should update
-        var s = $("img#selector-background").attr('src') + "?" + new Date().getTime();
-        $("img#selector-background").attr('src', s)
+                fetchData();
+                $selectorCanvasElem.off("mousemove mousedown");
+            })
+            .attr("src", screenshot_url);
+
+        let s = `${$selectorBackgroundElem.attr('src')}?${new Date().getTime()}`;
+        $selectorBackgroundElem.attr('src', s);
     }
 
-    // This is fired once the img src is loaded in bootstrap_visualselector()
-    function fetch_data() {
-        // Image is ready
-        $('.fetching-update-notice').html("Fetching element data..");
+    function alertIfFilterNotFound() {
+        let existingFilters = splitToList($includeFiltersElem.val());
+        let sizePosXpaths = selectorData['size_pos'].map(sel => sel.xpath);
+
+        for (let filter of existingFilters) {
+            if (!sizePosXpaths.includes(filter)) {
+                alert(`One or more of your existing filters was not found and will be removed when a new filter is selected.`);
+                break;
+            }
+        }
+    }
+
+    function fetchData() {
+        $fetchingUpdateNoticeElem.html("Fetching element data..");
 
         $.ajax({
             url: watch_visual_selector_data_url,
             context: document.body
-        }).done(function (data) {
-            $('.fetching-update-notice').html("Rendering..");
-            selector_data = data;
-            console.log("Reported browser width from backend: " + data['browser_width']);
-            state_clicked = false;
-            set_scale();
-            reflow_selector();
-            $('.fetching-update-notice').fadeOut();
-        });
+        }).done((data) => {
+            $fetchingUpdateNoticeElem.html("Rendering..");
+            selectorData = data;
+            sortScrapedElementsBySize();
+            console.log(`Reported browser width from backend: ${data['browser_width']}`);
 
+            // Little sanity check for the user, alert them if something missing
+            alertIfFilterNotFound();
+
+            setScale();
+            reflowSelector();
+            $fetchingUpdateNoticeElem.fadeOut();
+        });
     }
 
+    function updateFiltersText() {
+        // Assuming currentSelections is already defined and contains the selections
+        let uniqueSelections = new Set(currentSelections.map(sel => (sel[0] === '/' ? `xpath:${sel.xpath}` : sel.xpath)));
 
-    function set_scale() {
-
-        // some things to check if the scaling doesnt work
-        // - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq
-        $("#selector-wrapper").show();
-        selector_image = $("img#selector-background")[0];
-        selector_image_rect = selector_image.getBoundingClientRect();
-
-        // make the canvas the same size as the image
-        $('#selector-canvas').attr('height', selector_image_rect.height);
-        $('#selector-canvas').attr('width', selector_image_rect.width);
-        $('#selector-wrapper').attr('width', selector_image_rect.width);
-        x_scale = selector_image_rect.width / selector_data['browser_width'];
-        y_scale = selector_image_rect.height / selector_image.naturalHeight;
-        ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
-        ctx.fillStyle = 'rgba(255,0,0, 0.1)';
-        ctx.lineWidth = 3;
-        console.log("scaling set  x: " + x_scale + " by y:" + y_scale);
-        $("#selector-current-xpath").css('max-width', selector_image_rect.width);
-    }
-
-    function reflow_selector() {
-        $(window).resize(function () {
-            set_scale();
-            highlight_current_selected_i();
-        });
-        var selector_currnt_xpath_text = $("#selector-current-xpath span");
-
-        set_scale();
-
-        console.log(selector_data['size_pos'].length + " selectors found");
-
-        // highlight the default one if we can find it in the xPath list
-        // or the xpath matches the default one
-        found = false;
-        if (current_default_xpath.length) {
-            // Find the first one that matches
-            // @todo In the future paint all that match
-            for (const c of current_default_xpath) {
-                for (var i = selector_data['size_pos'].length; i !== 0; i--) {
-                    if (selector_data['size_pos'][i - 1].xpath.trim() === c.trim()) {
-                        console.log("highlighting " + c);
-                        current_selected_i = i - 1;
-                        highlight_current_selected_i();
-                        found = true;
-                        break;
-                    }
-                }
-                if (found) {
-                    break;
-                }
-            }
-            if (!found) {
-                alert("Unfortunately your existing CSS/xPath Filter was no longer found!");
-            }
+        if (currentSelections.length > 0) {
+            // Convert the Set back to an array and join with newline characters
+            let textboxFilterText = Array.from(uniqueSelections).join("\n");
+            $includeFiltersElem.val(textboxFilterText);
         }
+    }
 
+    function setScale() {
+        $selectorWrapperElem.show();
+        selectorImage = $selectorBackgroundElem[0];
+        selectorImageRect = selectorImage.getBoundingClientRect();
 
-        $('#selector-canvas').bind('mousemove', function (e) {
-            if (state_clicked) {
-                return;
+        $selectorCanvasElem.attr({
+            'height': selectorImageRect.height,
+            'width': selectorImageRect.width
+        });
+        $selectorWrapperElem.attr('width', selectorImageRect.width);
+        $('#visual-selector-heading').css('max-width', selectorImageRect.width + "px")
+
+        xScale = selectorImageRect.width / selectorImage.naturalWidth;
+        yScale = selectorImageRect.height / selectorImage.naturalHeight;
+
+        ctx.strokeStyle = STROKE_STYLE_HIGHLIGHT;
+        ctx.fillStyle = FILL_STYLE_REDLINE;
+        ctx.lineWidth = 3;
+        console.log("Scaling set  x: " + xScale + " by y:" + yScale);
+        $("#selector-current-xpath").css('max-width', selectorImageRect.width);
+    }
+
+    function reflowSelector() {
+        $(window).resize(() => {
+            setScale();
+            highlightCurrentSelected();
+        });
+
+        setScale();
+
+        console.log(selectorData['size_pos'].length + " selectors found");
+
+        let existingFilters = splitToList($includeFiltersElem.val());
+
+        selectorData['size_pos'].forEach(sel => {
+            if ((!runInClearMode && sel.highlight_as_custom_filter) || existingFilters.includes(sel.xpath)) {
+                console.log("highlighting " + c);
+                currentSelections.push(sel);
             }
-            ctx.clearRect(0, 0, c.width, c.height);
-            current_selected_i = null;
+        });
 
-            // Add in offset
-            if ((typeof e.offsetX === "undefined" || typeof e.offsetY === "undefined") || (e.offsetX === 0 && e.offsetY === 0)) {
-                var targetOffset = $(e.target).offset();
+
+        highlightCurrentSelected();
+        updateFiltersText();
+
+        $selectorCanvasElem.bind('mousemove', handleMouseMove.debounce(5));
+        $selectorCanvasElem.bind('mousedown', handleMouseDown.debounce(5));
+        $selectorCanvasElem.bind('mouseleave', highlightCurrentSelected.debounce(5));
+
+        function handleMouseMove(e) {
+            if (!e.offsetX && !e.offsetY) {
+                const targetOffset = $(e.target).offset();
                 e.offsetX = e.pageX - targetOffset.left;
                 e.offsetY = e.pageY - targetOffset.top;
             }
 
-            // Reverse order - the most specific one should be deeper/"laster"
-            // Basically, find the most 'deepest'
-            var found = 0;
-            ctx.fillStyle = 'rgba(205,0,0,0.35)';
-            // Will be sorted by smallest width*height first
-            for (var i = 0; i <= selector_data['size_pos'].length; i++) {
-                // draw all of them? let them choose somehow?
-                var sel = selector_data['size_pos'][i];
-                // If we are in a bounding-box
-                if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
-                    &&
-                    e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
+            ctx.fillStyle = FILL_STYLE_HIGHLIGHT;
 
-                ) {
-
-                    // FOUND ONE
-                    set_current_selected_text(sel.xpath);
-                    ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-                    ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-
-                    // no need to keep digging
-                    // @todo or, O to go out/up, I to go in
-                    // or double click to go up/out the selector?
-                    current_selected_i = i;
-                    found += 1;
-                    break;
+            selectorData['size_pos'].forEach(sel => {
+                if (e.offsetY > sel.top * yScale && e.offsetY < sel.top * yScale + sel.height * yScale &&
+                    e.offsetX > sel.left * yScale && e.offsetX < sel.left * yScale + sel.width * yScale) {
+                    setCurrentSelectedText(sel.xpath);
+                    drawHighlight(sel);
+                    currentSelections.push(sel);
+                    currentSelection = sel;
+                    highlightCurrentSelected();
+                    currentSelections.pop();
                 }
-            }
-
-        }.debounce(5));
-
-        function set_current_selected_text(s) {
-            selector_currnt_xpath_text[0].innerHTML = s;
-        }
-
-        function highlight_current_selected_i() {
-            if (state_clicked) {
-                state_clicked = false;
-                xctx.clearRect(0, 0, c.width, c.height);
-                return;
-            }
-
-            var sel = selector_data['size_pos'][current_selected_i];
-            if (sel[0] == '/') {
-                // @todo - not sure just checking / is right
-                $("#include_filters").val('xpath:' + sel.xpath);
-            } else {
-                $("#include_filters").val(sel.xpath);
-            }
-            xctx.fillStyle = 'rgba(205,205,205,0.95)';
-            xctx.strokeStyle = 'rgba(225,0,0,0.9)';
-            xctx.lineWidth = 3;
-            xctx.fillRect(0, 0, c.width, c.height);
-            // Clear out what only should be seen (make a clear/clean spot)
-            xctx.clearRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-            xctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-            state_clicked = true;
-            set_current_selected_text(sel.xpath);
-
+            })
         }
 
 
-        $('#selector-canvas').bind('mousedown', function (e) {
-            highlight_current_selected_i();
-        });
+        function setCurrentSelectedText(s) {
+            $selectorCurrentXpathElem[0].innerHTML = s;
+        }
+
+        function drawHighlight(sel) {
+            ctx.strokeRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
+            ctx.fillRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
+        }
+
+        function handleMouseDown() {
+            // If we are in 'appendToList' mode, grow the list, if not, just 1
+            currentSelections = appendToList ? [...currentSelections, currentSelection] : [currentSelection];
+            highlightCurrentSelected();
+            updateFiltersText();
+        }
+
     }
 
+    function highlightCurrentSelected() {
+        xctx.fillStyle = FILL_STYLE_GREYED_OUT;
+        xctx.strokeStyle = STROKE_STYLE_REDLINE;
+        xctx.lineWidth = 3;
+        xctx.clearRect(0, 0, c.width, c.height);
+
+        currentSelections.forEach(sel => {
+            //xctx.clearRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
+            xctx.strokeRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
+        });
+    }
 });

changedetectionio/static/styles/scss/parts/_visualselector.scss

@@ -1,6 +1,8 @@
 
 #selector-wrapper {
   height: 100%;
+  text-align: center;
+  
   max-height: 70vh;
   overflow-y: scroll;
   position: relative;

changedetectionio/static/styles/scss/styles.scss

@@ -671,14 +671,25 @@ footer {
   and also iPads specifically.
   */
   .watch-table {
+    /* make headings work on mobile */
+    thead {
+      display: block;
+      tr {
+        th {
+          display: inline-block;
+        }
+      }
+      .empty-cell {
+        display: none;
+      }
+    }
 
     /* Force table to not be like tables anymore */
-    thead,
-    tbody,
-    th,
-    td,
-    tr {
-      display: block;
+    tbody {
+      td,
+      tr {
+        display: block;
+      }
     }
 
     .last-checked {
@@ -702,13 +713,6 @@ footer {
       display: inline-block;
     }
 
-    /* Hide table headers (but not display: none;, for accessibility) */
-    thead tr {
-      position: absolute;
-      top: -9999px;
-      left: -9999px;
-    }
-
     .pure-table td,
     .pure-table th {
       border: none;
@@ -753,6 +757,7 @@ footer {
   thead {
     background-color: var(--color-background-table-thead);
     color: var(--color-text);
+    border-bottom: 1px solid var(--color-background-table-thead);
   }
 
   td,
@@ -1021,6 +1026,11 @@ ul {
   border-radius: 10px;
   margin-bottom: 1em;
   display: none;
+  button {
+    /* some space if they wrap the page */
+    margin-bottom: 3px;
+    margin-top: 3px;
+  }
 }
 
 .checkbox-uuid {

changedetectionio/static/styles/styles.css

@@ -863,14 +863,17 @@ footer {
   and also iPads specifically.
   */
   .watch-table {
+    /* make headings work on mobile */
     /* Force table to not be like tables anymore */
-    /* Force table to not be like tables anymore */
-    /* Hide table headers (but not display: none;, for accessibility) */ }
-    .watch-table thead,
-    .watch-table tbody,
-    .watch-table th,
-    .watch-table td,
-    .watch-table tr {
+    /* Force table to not be like tables anymore */ }
+    .watch-table thead {
+      display: block; }
+      .watch-table thead tr th {
+        display: inline-block; }
+      .watch-table thead .empty-cell {
+        display: none; }
+    .watch-table tbody td,
+    .watch-table tbody tr {
       display: block; }
     .watch-table .last-checked > span {
       vertical-align: middle; }
@@ -882,10 +885,6 @@ footer {
       content: "Last Changed "; }
     .watch-table td.inline {
       display: inline-block; }
-    .watch-table thead tr {
-      position: absolute;
-      top: -9999px;
-      left: -9999px; }
     .watch-table .pure-table td,
     .watch-table .pure-table th {
       border: none; }
@@ -912,7 +911,8 @@ footer {
   border-color: var(--color-border-table-cell); }
   .pure-table thead {
     background-color: var(--color-background-table-thead);
-    color: var(--color-text); }
+    color: var(--color-text);
+    border-bottom: 1px solid var(--color-background-table-thead); }
   .pure-table td,
   .pure-table th {
     border-left-color: var(--color-border-table-cell); }
@@ -1065,6 +1065,7 @@ ul {
 
 #selector-wrapper {
   height: 100%;
+  text-align: center;
   max-height: 70vh;
   overflow-y: scroll;
   position: relative; }
@@ -1127,6 +1128,10 @@ ul {
   border-radius: 10px;
   margin-bottom: 1em;
   display: none; }
+  #checkbox-operations button {
+    /* some space if they wrap the page */
+    margin-bottom: 3px;
+    margin-top: 3px; }
 
 .checkbox-uuid > * {
   vertical-align: middle; }

changedetectionio/store.py

@@ -124,12 +124,12 @@ class ChangeDetectionStore:
                 self.__data['app_guid'] = str(uuid_builder.uuid4())
 
         # Generate the URL access token for RSS feeds
-        if not 'rss_access_token' in self.__data['settings']['application']:
+        if not self.__data['settings']['application'].get('rss_access_token'):
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['rss_access_token'] = secret
 
         # Generate the API access token
-        if not 'api_access_token' in self.__data['settings']['application']:
+        if not self.__data['settings']['application'].get('api_access_token'):
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['api_access_token'] = secret
 
@@ -163,7 +163,6 @@ class ChangeDetectionStore:
                         del (update_obj[dict_key])
 
             self.__data['watching'][uuid].update(update_obj)
-
         self.needs_write = True
 
     @property
@@ -178,7 +177,7 @@ class ChangeDetectionStore:
     @property
     def has_unviewed(self):
         for uuid, watch in self.__data['watching'].items():
-            if watch.viewed == False:
+            if watch.history_n >= 2 and watch.viewed == False:
                 return True
         return False
 
@@ -243,6 +242,14 @@ class ChangeDetectionStore:
     def clear_watch_history(self, uuid):
         import pathlib
 
+        # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc
+        for item in pathlib.Path(os.path.join(self.datastore_path, uuid)).rglob("*.*"):
+            unlink(item)
+
+        # Force the attr to recalculate
+        bump = self.__data['watching'][uuid].history
+
+        # Do this last because it will trigger a recheck due to last_checked being zero
         self.__data['watching'][uuid].update({
                 'browser_steps_last_error_step' : None,
                 'check_count': 0,
@@ -259,13 +266,6 @@ class ChangeDetectionStore:
                 'track_ldjson_price_data': None,
             })
 
-        # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc
-        for item in pathlib.Path(os.path.join(self.datastore_path, uuid)).rglob("*.*"):
-            unlink(item)
-
-        # Force the attr to recalculate
-        bump = self.__data['watching'][uuid].history
-
         self.needs_write_urgent = True
 
     def add_watch(self, url, tag='', extras=None, tag_uuids=None, write_to_disk_now=True):
@@ -376,46 +376,6 @@ class ChangeDetectionStore:
 
         return False
 
-    # Save as PNG, PNG is larger but better for doing visual diff in the future
-    def save_screenshot(self, watch_uuid, screenshot: bytes, as_error=False):
-        if not self.data['watching'].get(watch_uuid):
-            return
-
-        if as_error:
-            target_path = os.path.join(self.datastore_path, watch_uuid, "last-error-screenshot.png")
-        else:
-            target_path = os.path.join(self.datastore_path, watch_uuid, "last-screenshot.png")
-
-        self.data['watching'][watch_uuid].ensure_data_dir_exists()
-
-        with open(target_path, 'wb') as f:
-            f.write(screenshot)
-            f.close()
-
-
-    def save_error_text(self, watch_uuid, contents):
-        if not self.data['watching'].get(watch_uuid):
-            return
-
-        self.data['watching'][watch_uuid].ensure_data_dir_exists()
-        target_path = os.path.join(self.datastore_path, watch_uuid, "last-error.txt")
-        with open(target_path, 'w') as f:
-            f.write(contents)
-
-    def save_xpath_data(self, watch_uuid, data, as_error=False):
-
-        if not self.data['watching'].get(watch_uuid):
-            return
-        if as_error:
-            target_path = os.path.join(self.datastore_path, watch_uuid, "elements-error.json")
-        else:
-            target_path = os.path.join(self.datastore_path, watch_uuid, "elements.json")
-        self.data['watching'][watch_uuid].ensure_data_dir_exists()
-        with open(target_path, 'w') as f:
-            f.write(json.dumps(data))
-            f.close()
-
-
     def sync_to_json(self):
         logger.info("Saving JSON..")
         try:
@@ -884,3 +844,8 @@ class ChangeDetectionStore:
                 # Something custom here
                 self.__data["watching"][uuid]['time_between_check_use_default'] = False
 
+    # Correctly set datatype for older installs where 'tag' was string and update_12 did not catch it
+    def update_16(self):
+        for uuid, watch in self.data['watching'].items():
+            if isinstance(watch.get('tags'), str):
+                self.data['watching'][uuid]['tags'] = []

changedetectionio/templates/base.html

@@ -26,7 +26,11 @@
     <meta name="msapplication-TileColor" content="#da532c">
     <meta name="msapplication-config" content="favicons/browserconfig.xml">
     <meta name="theme-color" content="#ffffff">
+    <script>
+        const csrftoken="{{ csrf_token() }}";
+    </script>
     <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
+    <script src="{{url_for('static_content', group='js', filename='csrf.js')}}" defer></script>
   </head>
 
   <body>

changedetectionio/templates/edit.html

@@ -292,7 +292,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                             <ul>
                                 <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
                                 {% if jq_support %}
-                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
+                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>. Prefix <code>jqraw:</code> outputs the results as text instead of a JSON list.</li>
                                 {% else %}
                                 <li>jq support not installed</li>
                                 {% endif %}
@@ -432,9 +432,8 @@ Unavailable") }}
                 <fieldset>
                     <div class="pure-control-group">
                         {% if visualselector_enabled %}
-                            <span class="pure-form-message-inline">
-                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection &dash; after the <i>Browser Steps</i> has completed.<br>
-                                This tool is a helper to manage filters in the  "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab.
+                            <span class="pure-form-message-inline" id="visual-selector-heading">
+                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection. It automatically fills-in the filters in the "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab. Use <strong>Shift+Click</strong> to select multiple items.
                             </span>
 
                             <div id="selector-header">

changedetectionio/templates/settings.html

@@ -62,6 +62,9 @@
                         <span class="pure-form-message-inline">Allow access to view watch diff page when password is enabled (Good for sharing the diff page)
                         </span>
                     </div>
+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }}
+                    </div>
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.pager_size) }}
                         <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>

changedetectionio/templates/watch-overview.html

@@ -68,11 +68,11 @@
                 {% set link_order = "desc" if sort_order  == 'asc' else "asc" %}
                 {% set arrow_span = "" %}
                 <th><input style="vertical-align: middle" type="checkbox" id="check-all" > <a class="{{ 'active '+link_order if sort_attribute == 'date_created' else 'inactive' }}"  href="{{url_for('index', sort='date_created', order=link_order, tag=active_tag_uuid)}}"># <span class='arrow {{link_order}}'></span></a></th>
-                <th></th>
+                <th class="empty-cell"></th>
                 <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('index', sort='label', order=link_order, tag=active_tag_uuid)}}">Website <span class='arrow {{link_order}}'></span></a></th>
                 <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order, tag=active_tag_uuid)}}">Last Checked <span class='arrow {{link_order}}'></span></a></th>
                 <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order, tag=active_tag_uuid)}}">Last Changed <span class='arrow {{link_order}}'></span></a></th>
-                <th></th>
+                <th class="empty-cell"></th>
             </tr>
             </thead>
             <tbody>

changedetectionio/tests/conftest.py

@@ -1,4 +1,7 @@
 #!/usr/bin/python3
+import resource
+import time
+from threading import Thread
 
 import pytest
 from changedetectionio import changedetection_app
@@ -23,6 +26,36 @@ def reportlog(pytestconfig):
     yield
     logger.remove(handler_id)
 
+
+def track_memory(memory_usage, ):
+    while not memory_usage["stop"]:
+        max_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+        memory_usage["peak"] = max(memory_usage["peak"], max_rss)
+        time.sleep(0.01)  # Adjust the sleep time as needed
+
+@pytest.fixture(scope='function')
+def measure_memory_usage(request):
+    memory_usage = {"peak": 0, "stop": False}
+    tracker_thread = Thread(target=track_memory, args=(memory_usage,))
+    tracker_thread.start()
+
+    yield
+
+    memory_usage["stop"] = True
+    tracker_thread.join()
+
+    # Note: ru_maxrss is in kilobytes on Unix-based systems
+    max_memory_used = memory_usage["peak"] / 1024  # Convert to MB
+    s = f"Peak memory used by the test {request.node.fspath} - '{request.node.name}': {max_memory_used:.2f} MB"
+    logger.debug(s)
+
+    with open("test-memory.log", 'a') as f:
+        f.write(f"{s}\n")
+
+    # Assert that the memory usage is less than 200MB
+    assert max_memory_used < 150, f"Memory usage exceeded 200MB: {max_memory_used:.2f} MB"
+
+
 def cleanup(datastore_path):
     import glob
     # Unlink test output files

changedetectionio/tests/custom_browser_url/test_custom_browser_url.py

@@ -77,13 +77,13 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
 
 
 # Requires playwright to be installed
-def test_request_via_custom_browser_url(client, live_server):
+def test_request_via_custom_browser_url(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
     # We do this so we can grep the logs of the custom container and see if the request actually went through that container
     do_test(client, live_server, make_test_use_extra_browser=True)
 
 
-def test_request_not_via_custom_browser_url(client, live_server):
+def test_request_not_via_custom_browser_url(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
     # We do this so we can grep the logs of the custom container and see if the request actually went through that container
     do_test(client, live_server, make_test_use_extra_browser=False)

changedetectionio/tests/fetchers/test_content.py

@@ -6,7 +6,7 @@ from ..util import live_server_setup, wait_for_all_checks
 import logging
 
 # Requires playwright to be installed
-def test_fetch_webdriver_content(client, live_server):
+def test_fetch_webdriver_content(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
     #####################

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -3,7 +3,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 
-def test_execute_custom_js(client, live_server):
+def test_execute_custom_js(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks
 
 
-def test_preferred_proxy(client, live_server):
+def test_preferred_proxy(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
     url = "http://chosen.changedetection.io"
 

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 
-def test_noproxy_option(client, live_server):
+def test_noproxy_option(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
     # Run by run_proxy_tests.sh
     # Call this URL then scan the containers that it never went through them

changedetectionio/tests/proxy_list/test_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 # just make a request, we will grep in the docker logs to see it actually got called
-def test_check_basic_change_detection_functionality(client, live_server):
+def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
     res = client.post(
         url_for("import_page"),

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -6,7 +6,7 @@ from ..util import live_server_setup, wait_for_all_checks
 import os
 
 # just make a request, we will grep in the docker logs to see it actually got called
-def test_select_custom(client, live_server):
+def test_select_custom(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
     # Goto settings, add our custom one

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from changedetectionio.tests.util import live_server_setup, wait_for_all_checks
 
 
-def test_socks5(client, live_server):
+def test_socks5(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
     # Setup a proxy

changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py

@@ -7,7 +7,7 @@ from changedetectionio.tests.util import live_server_setup, wait_for_all_checks
 
 # should be proxies.json mounted from run_proxy_tests.sh already
 # -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json
-def test_socks5_from_proxiesjson_file(client, live_server):
+def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
     test_url = "https://changedetection.io/CHANGELOG.txt?socks-test-tag=" + os.getenv('SOCKSTEST', '')

changedetectionio/tests/restock/test_restock.py

@@ -48,7 +48,7 @@ def set_back_in_stock_response():
     return None
 
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
-def test_restock_detection(client, live_server):
+def test_restock_detection(client, live_server, measure_memory_usage):
 
     set_original_response()
     #assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"

changedetectionio/tests/smtp/smtp-test-server.py

@@ -1,42 +1,51 @@
 #!/usr/bin/python3
-import smtpd
-import asyncore
+import asyncio
+from aiosmtpd.controller import Controller
+from aiosmtpd.smtp import SMTP
 
 # Accept a SMTP message and offer a way to retrieve the last message via TCP Socket
 
 last_received_message = b"Nothing"
 
 
-class CustomSMTPServer(smtpd.SMTPServer):
-
-    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
+class CustomSMTPHandler:
+    async def handle_DATA(self, server, session, envelope):
         global last_received_message
-        last_received_message = data
-        print('Receiving message from:', peer)
-        print('Message addressed from:', mailfrom)
-        print('Message addressed to  :', rcpttos)
-        print('Message length        :', len(data))
-        print(data.decode('utf8'))
-        return
+        last_received_message = envelope.content
+        print('Receiving message from:', session.peer)
+        print('Message addressed from:', envelope.mail_from)
+        print('Message addressed to  :', envelope.rcpt_tos)
+        print('Message length        :', len(envelope.content))
+        print(envelope.content.decode('utf8'))
+        return '250 Message accepted for delivery'
 
 
-# Just print out the last message received on plain TCP socket server
-class EchoServer(asyncore.dispatcher):
-
-    def __init__(self, host, port):
-        asyncore.dispatcher.__init__(self)
-        self.create_socket()
-        self.set_reuse_addr()
-        self.bind((host, port))
-        self.listen(5)
-
-    def handle_accepted(self, sock, addr):
+class EchoServerProtocol(asyncio.Protocol):
+    def connection_made(self, transport):
         global last_received_message
-        print('Incoming connection from %s' % repr(addr))
-        sock.send(last_received_message)
+        self.transport = transport
+        peername = transport.get_extra_info('peername')
+        print('Incoming connection from {}'.format(peername))
+        self.transport.write(last_received_message)
+
         last_received_message = b''
+        self.transport.close()
 
 
-server = CustomSMTPServer(('0.0.0.0', 11025), None)  # SMTP mail goes here
-server2 = EchoServer('0.0.0.0', 11080)  # Echo back last message received
-asyncore.loop()
+async def main():
+    # Start the SMTP server
+    controller = Controller(CustomSMTPHandler(), hostname='0.0.0.0', port=11025)
+    controller.start()
+
+    # Start the TCP Echo server
+    loop = asyncio.get_running_loop()
+    server = await loop.create_server(
+        lambda: EchoServerProtocol(),
+        '0.0.0.0', 11080
+    )
+    async with server:
+        await server.serve_forever()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -32,13 +32,15 @@ def get_last_message_from_smtp_server():
     client_socket.connect((smtp_test_server, port))  # connect to the server
 
     data = client_socket.recv(50024).decode()  # receive response
+    logging.info("get_last_message_from_smtp_server..")
+    logging.info(data)
     client_socket.close()  # close the connection
     return data
 
 
 # Requires running the test SMTP server
 
-def test_check_notification_email_formats_default_HTML(client, live_server):
+def test_check_notification_email_formats_default_HTML(client, live_server, measure_memory_usage):
     # live_server_setup(live_server)
     set_original_response()
 
@@ -71,6 +73,8 @@ def test_check_notification_email_formats_default_HTML(client, live_server):
 
     wait_for_all_checks(client)
     set_longer_modified_response()
+    time.sleep(2)
+
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -81,14 +85,14 @@ def test_check_notification_email_formats_default_HTML(client, live_server):
 
     # The email should have two bodies, and the text/html part should be <br>
     assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
+    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
     assert 'Content-Type: text/html' in msg
     assert '(added) So let\'s see what happens.<br>' in msg  # the html part
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 
-def test_check_notification_email_formats_default_Text_override_HTML(client, live_server):
+def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage):
     # live_server_setup(live_server)
 
     # HTML problems? see this
@@ -135,6 +139,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
     wait_for_all_checks(client)
     set_longer_modified_response()
+    time.sleep(2)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -147,7 +152,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     # The email should not have two bodies, should be TEXT only
 
     assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
+    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
 
     set_original_response()
     # Now override as HTML format
@@ -168,7 +173,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
     # The email should have two bodies, and the text/html part should be <br>
     assert 'Content-Type: text/plain' in msg
-    assert '(removed) So let\'s see what happens.\n' in msg  # The plaintext part with \n
+    assert '(removed) So let\'s see what happens.\r\n' in msg  # The plaintext part with \n
     assert 'Content-Type: text/html' in msg
     assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
 

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -35,10 +35,10 @@ def set_original(excluding=None, add_line=None):
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-def test_check_removed_line_contains_trigger(client, live_server):
+def test_check_removed_line_contains_trigger(client, live_server, measure_memory_usage):
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -103,7 +103,7 @@ def test_check_removed_line_contains_trigger(client, live_server):
     assert b'Deleted' in res.data
 
 
-def test_check_add_line_contains_trigger(client, live_server):
+def test_check_add_line_contains_trigger(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     # Give the endpoint time to spin up

changedetectionio/tests/test_api.py

@@ -53,10 +53,10 @@ def is_valid_uuid(val):
         return False
 
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-def test_api_simple(client, live_server):
+def test_api_simple(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     api_key = extract_api_key_from_UI(client)
@@ -149,6 +149,15 @@ def test_api_simple(client, live_server):
         headers={'x-api-key': api_key},
     )
     assert b'which has this one new line' in res.data
+    assert b'<div id' not in res.data
+
+    # Fetch the HTML of the latest one
+    res = client.get(
+        url_for("watchsinglehistory", uuid=watch_uuid, timestamp='latest')+"?html=1",
+        headers={'x-api-key': api_key},
+    )
+    assert b'which has this one new line' in res.data
+    assert b'<div id' in res.data
 
     # Fetch the whole watch
     res = client.get(
@@ -232,7 +241,7 @@ def test_api_simple(client, live_server):
     )
     assert len(res.json) == 0, "Watch list should be empty"
 
-def test_access_denied(client, live_server):
+def test_access_denied(client, live_server, measure_memory_usage):
     # `config_api_token_enabled` Should be On by default
     res = client.get(
         url_for("createwatch")
@@ -278,7 +287,7 @@ def test_access_denied(client, live_server):
     )
     assert b"Settings updated." in res.data
 
-def test_api_watch_PUT_update(client, live_server):
+def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
 
     #live_server_setup(live_server)
     api_key = extract_api_key_from_UI(client)
@@ -360,7 +369,7 @@ def test_api_watch_PUT_update(client, live_server):
     assert b'Deleted' in res.data
 
 
-def test_api_import(client, live_server):
+def test_api_import(client, live_server, measure_memory_usage):
     api_key = extract_api_key_from_UI(client)
 
     res = client.post(

changedetectionio/tests/test_auth.py

@@ -2,13 +2,12 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 
-def test_basic_auth(client, live_server):
+
+def test_basic_auth(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_basicauth_method', _external=True).replace("//","//myuser:mypass@")
@@ -19,8 +18,8 @@ def test_basic_auth(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
     time.sleep(1)
-
     # Check form validation
     res = client.post(
         url_for("edit_page", uuid="first"),
@@ -29,7 +28,7 @@ def test_basic_auth(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    time.sleep(1)
+    wait_for_all_checks(client)
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -76,11 +76,11 @@ def set_response_without_ldjson():
         f.write(test_return_data)
     return None
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
 # actually only really used by the distll.io importer, but could be handy too
-def test_check_ldjson_price_autodetect(client, live_server):
+def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage):
 
     set_response_with_ldjson()
 
@@ -100,11 +100,12 @@ def test_check_ldjson_price_autodetect(client, live_server):
 
     # Accept it
     uuid = extract_UUID_from_client(client)
-
+    time.sleep(1)
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
     wait_for_all_checks(client)
 
     # Trigger a check
+    time.sleep(1)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     # Offer should be gone
@@ -166,7 +167,7 @@ def _test_runner_check_bad_format_ignored(live_server, client, has_ldjson_price_
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
 
-def test_bad_ldjson_is_correctly_ignored(client, live_server):
+def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     test_return_data = """
             <html>

changedetectionio/tests/test_backend.py

@@ -3,7 +3,8 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
+    extract_UUID_from_client
 
 sleep_time_for_fetch_thread = 3
 
@@ -16,7 +17,7 @@ def test_inscriptus():
     assert stripped_text_from_html == 'test!\nok man'
 
 
-def test_check_basic_change_detection_functionality(client, live_server):
+def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
     set_original_response()
     live_server_setup(live_server)
 
@@ -62,9 +63,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
     # Make a change
     set_modified_response()
 
-    res = urlopen(url_for('test_endpoint', _external=True))
-    assert b'which has this one new line' in res.read()
-
     # Force recheck
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches queued for rechecking.' in res.data
@@ -135,12 +133,23 @@ def test_check_basic_change_detection_functionality(client, live_server):
     # It should have picked up the <title>
     assert b'head title' in res.data
 
+    # Be sure the last_viewed is going to be greater than the last snapshot
+    time.sleep(1)
+
     # hit the mark all viewed link
     res = client.get(url_for("mark_all_viewed"), follow_redirects=True)
 
     assert b'Mark all viewed' not in res.data
     assert b'unviewed' not in res.data
 
+    # #2458 "clear history" should make the Watch object update its status correctly when the first snapshot lands again
+    uuid = extract_UUID_from_client(client)
+    client.get(url_for("clear_watch_history", uuid=uuid))
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+    res = client.get(url_for("index"))
+    assert b'preview/' in res.data
+
     #
     # Cleanup everything
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_backup.py

@@ -8,7 +8,7 @@ import re
 import time
 
 
-def test_backup(client, live_server):
+def test_backup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
     set_original_response()

changedetectionio/tests/test_block_while_text_present.py

@@ -60,7 +60,7 @@ def set_modified_response_minus_block_text():
         f.write(test_return_data)
 
 
-def test_check_block_changedetection_text_NOT_present(client, live_server):
+def test_check_block_changedetection_text_NOT_present(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
     # Use a mix of case in ZzZ to prove it works case-insensitive.

changedetectionio/tests/test_clone.py

@@ -6,7 +6,7 @@ from . util import live_server_setup
 
 
 
-def test_trigger_functionality(client, live_server):
+def test_trigger_functionality(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
 

changedetectionio/tests/test_css_selector.py

@@ -70,7 +70,7 @@ def test_include_filters_output():
 
 
 # Tests the whole stack works with the CSS Filter
-def test_check_markup_include_filters_restriction(client, live_server):
+def test_check_markup_include_filters_restriction(client, live_server, measure_memory_usage):
     sleep_time_for_fetch_thread = 3
 
     include_filters = "#sametext"
@@ -124,7 +124,7 @@ def test_check_markup_include_filters_restriction(client, live_server):
 
 
 # Tests the whole stack works with the CSS Filter
-def test_check_multiple_filters(client, live_server):
+def test_check_multiple_filters(client, live_server, measure_memory_usage):
     sleep_time_for_fetch_thread = 3
 
     include_filters = "#blob-a\r\nxpath://*[contains(@id,'blob-b')]"
@@ -180,7 +180,7 @@ def test_check_multiple_filters(client, live_server):
 # The filter exists, but did not contain anything useful
 # Mainly used when the filter contains just an IMG, this can happen when someone selects an image in the visual-selector
 # Tests fetcher can throw a "ReplyWithContentButNoText" exception after applying filter and extracting text
-def test_filter_is_empty_help_suggestion(client, live_server):
+def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     include_filters = "#blob-a"

changedetectionio/tests/test_element_removal.py

@@ -106,7 +106,7 @@ across multiple lines
     )
 
 
-def test_element_removal_full(client, live_server):
+def test_element_removal_full(client, live_server, measure_memory_usage):
     sleep_time_for_fetch_thread = 3
 
     set_original_response()

changedetectionio/tests/test_encoding.py

@@ -3,7 +3,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 import pytest
 
 
@@ -24,12 +24,9 @@ def set_html_response():
 
 
 # In the case the server does not issue a charset= or doesnt have content_type header set
-def test_check_encoding_detection(client, live_server):
+def test_check_encoding_detection(client, live_server, measure_memory_usage):
     set_html_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', content_type="text/html", _external=True)
     client.post(
@@ -39,7 +36,7 @@ def test_check_encoding_detection(client, live_server):
     )
 
     # Give the thread time to pick it up
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     res = client.get(
         url_for("preview_page", uuid="first"),
@@ -53,12 +50,9 @@ def test_check_encoding_detection(client, live_server):
 
 
 # In the case the server does not issue a charset= or doesnt have content_type header set
-def test_check_encoding_detection_missing_content_type_header(client, live_server):
+def test_check_encoding_detection_missing_content_type_header(client, live_server, measure_memory_usage):
     set_html_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     client.post(
@@ -67,8 +61,7 @@ def test_check_encoding_detection_missing_content_type_header(client, live_serve
         follow_redirects=True
     )
 
-    # Give the thread time to pick it up
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     res = client.get(
         url_for("preview_page", uuid="first"),

changedetectionio/tests/test_errorhandling.py

@@ -54,7 +54,7 @@ def _runner_test_http_errors(client, live_server, http_code, expected_text):
     assert b'Deleted' in res.data
 
 
-def test_http_error_handler(client, live_server):
+def test_http_error_handler(client, live_server, measure_memory_usage):
     _runner_test_http_errors(client, live_server, 403, 'Access denied')
     _runner_test_http_errors(client, live_server, 404, 'Page not found')
     _runner_test_http_errors(client, live_server, 500, '(Internal server error) received')
@@ -63,7 +63,7 @@ def test_http_error_handler(client, live_server):
     assert b'Deleted' in res.data
 
 # Just to be sure error text is properly handled
-def test_DNS_errors(client, live_server):
+def test_DNS_errors(client, live_server, measure_memory_usage):
     # Give the endpoint time to spin up
     time.sleep(1)
 
@@ -87,7 +87,7 @@ def test_DNS_errors(client, live_server):
     assert b'Deleted' in res.data
 
 # Re 1513
-def test_low_level_errors_clear_correctly(client, live_server):
+def test_low_level_errors_clear_correctly(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     # Give the endpoint time to spin up
     time.sleep(1)

changedetectionio/tests/test_extract_csv.py

@@ -9,7 +9,7 @@ sleep_time_for_fetch_thread = 3
 
 
 
-def test_check_extract_text_from_diff(client, live_server):
+def test_check_extract_text_from_diff(client, live_server, measure_memory_usage):
     import time
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("Now it's {} seconds since epoch, time flies!".format(str(time.time())))
@@ -29,6 +29,7 @@ def test_check_extract_text_from_diff(client, live_server):
     # Load in 5 different numbers/changes
     last_date=""
     for n in range(5):
+        time.sleep(1)
         # Give the thread time to pick it up
         print("Bumping snapshot and checking.. ", n)
         last_date = str(time.time())

changedetectionio/tests/test_extract_regex.py

@@ -67,10 +67,10 @@ def set_multiline_response():
     return None
 
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-def test_check_filter_multiline(client, live_server):
+def test_check_filter_multiline(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     set_multiline_response()
 
@@ -122,7 +122,7 @@ def test_check_filter_multiline(client, live_server):
     # but the last one, which also says 'lines' shouldnt be here (non-greedy match checking)
     assert b'aaand something lines' not in res.data
 
-def test_check_filter_and_regex_extract(client, live_server):
+def test_check_filter_and_regex_extract(client, live_server, measure_memory_usage):
     
     include_filters = ".changetext"
 
@@ -205,7 +205,7 @@ def test_check_filter_and_regex_extract(client, live_server):
 
 
 
-def test_regex_error_handling(client, live_server):
+def test_regex_error_handling(client, live_server, measure_memory_usage):
 
     #live_server_setup(live_server)
 

changedetectionio/tests/test_filter_exist_changes.py

@@ -41,7 +41,7 @@ def set_response_with_filter():
         f.write(test_return_data)
     return None
 
-def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_server):
+def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_server, measure_memory_usage):
 #  Filter knowingly doesn't exist, like someone setting up a known filter to see if some cinema tickets are on sale again
 #  And the page has that filter available
 #  Then I should get a notification

changedetectionio/tests/test_filter_failure_notification.py

@@ -21,10 +21,11 @@ def set_response_with_filter():
         f.write(test_return_data)
     return None
 
-def run_filter_test(client, content_filter):
+def run_filter_test(client, live_server, content_filter):
+
+    # Response WITHOUT the filter ID element
+    set_original_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
     # cleanup for the next
     client.get(
         url_for("form_delete", uuid="all"),
@@ -79,6 +80,7 @@ def run_filter_test(client, content_filter):
         "include_filters": content_filter,
         "fetch_backend": "html_requests"})
 
+    # A POST here will also reset the filter failure counter (filter_failure_notification_threshold_attempts)
     res = client.post(
         url_for("edit_page", uuid="first"),
         data=notification_form_data,
@@ -91,20 +93,21 @@ def run_filter_test(client, content_filter):
     # Now the notification should not exist, because we didnt reach the threshold
     assert not os.path.isfile("test-datastore/notification.txt")
 
-    # -2 because we would have checked twice above (on adding and on edit)
+    # recheck it up to just before the threshold, including the fact that in the previous POST it would have rechecked (and incremented)
     for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT-2):
-        res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        client.get(url_for("form_watch_checknow"), follow_redirects=True)
         wait_for_all_checks(client)
-        assert not os.path.isfile("test-datastore/notification.txt"), f"test-datastore/notification.txt should not exist - Attempt {i}"
+        time.sleep(2) # delay for apprise to fire
+        assert not os.path.isfile("test-datastore/notification.txt"), f"test-datastore/notification.txt should not exist - Attempt {i} when threshold is {App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT}"
 
     # We should see something in the frontend
+    res = client.get(url_for("index"))
     assert b'Warning, no filters were found' in res.data
 
-    # One more check should trigger it (see -2 above)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    # One more check should trigger the _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT threshold
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
+    time.sleep(2)  # delay for apprise to fire
     # Now it should exist and contain our "filter not found" alert
     assert os.path.isfile("test-datastore/notification.txt")
 
@@ -148,14 +151,10 @@ def run_filter_test(client, content_filter):
 def test_setup(live_server):
     live_server_setup(live_server)
 
-def test_check_include_filters_failure_notification(client, live_server):
-    set_original_response()
-    wait_for_all_checks(client)
-    run_filter_test(client, '#nope-doesnt-exist')
+def test_check_include_filters_failure_notification(client, live_server, measure_memory_usage):
+    run_filter_test(client, live_server,'#nope-doesnt-exist')
 
-def test_check_xpath_filter_failure_notification(client, live_server):
-    set_original_response()
-    time.sleep(1)
-    run_filter_test(client, '//*[@id="nope-doesnt-exist"]')
+def test_check_xpath_filter_failure_notification(client, live_server, measure_memory_usage):
+    run_filter_test(client, live_server, '//*[@id="nope-doesnt-exist"]')
 
 # Test that notification is never sent

changedetectionio/tests/test_group.py

@@ -6,7 +6,7 @@ from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from
 import os
 
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
 def set_original_response():
@@ -39,7 +39,7 @@ def set_modified_response():
         f.write(test_return_data)
     return None
 
-def test_setup_group_tag(client, live_server):
+def test_setup_group_tag(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     set_original_response()
 
@@ -130,7 +130,7 @@ def test_setup_group_tag(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_tag_import_singular(client, live_server):
+def test_tag_import_singular(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     test_url = url_for('test_endpoint', _external=True)
@@ -150,7 +150,7 @@ def test_tag_import_singular(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_tag_add_in_ui(client, live_server):
+def test_tag_add_in_ui(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 #
     res = client.post(
@@ -167,7 +167,7 @@ def test_tag_add_in_ui(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_group_tag_notification(client, live_server):
+def test_group_tag_notification(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     set_original_response()
 
@@ -235,7 +235,7 @@ def test_group_tag_notification(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_limit_tag_ui(client, live_server):
+def test_limit_tag_ui(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     test_url = url_for('test_endpoint', _external=True)
@@ -273,7 +273,7 @@ def test_limit_tag_ui(client, live_server):
     assert b'Deleted' in res.data
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)
     assert b'All tags deleted' in res.data
-def test_clone_tag_on_import(client, live_server):
+def test_clone_tag_on_import(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -298,7 +298,7 @@ def test_clone_tag_on_import(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_clone_tag_on_quickwatchform_add(client, live_server):
+def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     test_url = url_for('test_endpoint', _external=True)
@@ -328,7 +328,7 @@ def test_clone_tag_on_quickwatchform_add(client, live_server):
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)
     assert b'All tags deleted' in res.data
 
-def test_order_of_filters_tag_filter_and_watch_filter(client, live_server):
+def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measure_memory_usage):
 
     # Add a tag with some config, import a tag and it should roughly work
     res = client.post(

changedetectionio/tests/test_history_consistency.py

@@ -5,15 +5,13 @@ import os
 import json
 import logging
 from flask import url_for
-from .util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 from urllib.parse import urlparse, parse_qs
 
-def test_consistent_history(client, live_server):
+def test_consistent_history(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-    r = range(1, 50)
+    r = range(1, 30)
 
     for one in r:
         test_url = url_for('test_endpoint', content_type="text/html", content=str(one), _external=True)
@@ -25,15 +23,8 @@ def test_consistent_history(client, live_server):
 
         assert b"1 Imported" in res.data
 
-    time.sleep(3)
-    while True:
-        res = client.get(url_for("index"))
-        logging.debug("Waiting for 'Checking now' to go away..")
-        if b'Checking now' not in res.data:
-            break
-        time.sleep(0.5)
+    wait_for_all_checks(client)
 
-    time.sleep(3)
     # Essentially just triggers the DB write/update
     res = client.post(
         url_for("settings_page"),
@@ -44,8 +35,9 @@ def test_consistent_history(client, live_server):
     )
     assert b"Settings updated." in res.data
 
-    # Give it time to write it out
-    time.sleep(3)
+
+    time.sleep(2)
+
     json_db_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, 'url-watches.json')
 
     json_obj = None
@@ -58,7 +50,7 @@ def test_consistent_history(client, live_server):
     # each one should have a history.txt containing just one line
     for w in json_obj['watching'].keys():
         history_txt_index_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, 'history.txt')
-        assert os.path.isfile(history_txt_index_file), "History.txt should exist where I expect it - {}".format(history_txt_index_file)
+        assert os.path.isfile(history_txt_index_file), f"History.txt should exist where I expect it at {history_txt_index_file}"
 
         # Same like in model.Watch
         with open(history_txt_index_file, "r") as f:
@@ -70,15 +62,15 @@ def test_consistent_history(client, live_server):
                                                      w))
         # Find the snapshot one
         for fname in files_in_watch_dir:
-            if fname != 'history.txt':
+            if fname != 'history.txt' and 'html' not in fname:
                 # contents should match what we requested as content returned from the test url
                 with open(os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, fname), 'r') as snapshot_f:
                     contents = snapshot_f.read()
                     watch_url = json_obj['watching'][w]['url']
                     u = urlparse(watch_url)
                     q = parse_qs(u[4])
-                    assert q['content'][0] == contents.strip(), "Snapshot file {} should contain {}".format(fname, q['content'][0])
+                    assert q['content'][0] == contents.strip(), f"Snapshot file {fname} should contain {q['content'][0]}"
 
 
 
-        assert len(files_in_watch_dir) == 2, "Should be just two files in the dir, history.txt and the snapshot"
+        assert len(files_in_watch_dir) == 3, "Should be just three files in the dir, html.br snapshot, history.txt and the extracted text snapshot"

changedetectionio/tests/test_ignore_text.py

@@ -82,7 +82,7 @@ def set_modified_ignore_response():
         f.write(test_return_data)
 
 
-def test_check_ignore_text_functionality(client, live_server):
+def test_check_ignore_text_functionality(client, live_server, measure_memory_usage):
 
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"
@@ -164,7 +164,7 @@ def test_check_ignore_text_functionality(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_check_global_ignore_text_functionality(client, live_server):
+def test_check_global_ignore_text_functionality(client, live_server, measure_memory_usage):
 
     # Give the endpoint time to spin up
     time.sleep(1)

changedetectionio/tests/test_ignorehighlighter.py

@@ -23,7 +23,7 @@ def set_original_ignore_response():
         f.write(test_return_data)
 
 
-def test_highlight_ignore(client, live_server):
+def test_highlight_ignore(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
     set_original_ignore_response()
     test_url = url_for('test_endpoint', _external=True)
@@ -45,7 +45,6 @@ def test_highlight_ignore(client, live_server):
     )
 
     res = client.get(url_for("edit_page", uuid=uuid))
-
     # should be a regex now
     assert b'/oh\ yeah\ \d+/' in res.data
 
@@ -55,3 +54,7 @@ def test_highlight_ignore(client, live_server):
     # And it should register in the preview page
     res = client.get(url_for("preview_page", uuid=uuid))
     assert b'<div class="ignored">oh yeah 456' in res.data
+
+    # Should be in base.html
+    assert b'csrftoken' in res.data
+

changedetectionio/tests/test_ignorehyperlinks.py

@@ -40,7 +40,7 @@ def set_modified_ignore_response():
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
-def test_render_anchor_tag_content_true(client, live_server):
+def test_render_anchor_tag_content_true(client, live_server, measure_memory_usage):
     """Testing that the link changes are detected when
     render_anchor_tag_content setting is set to true"""
     sleep_time_for_fetch_thread = 3

changedetectionio/tests/test_ignorestatuscode.py

@@ -39,7 +39,7 @@ def set_some_changed_response():
         f.write(test_return_data)
 
 
-def test_normal_page_check_works_with_ignore_status_code(client, live_server):
+def test_normal_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage):
 
 
     # Give the endpoint time to spin up
@@ -85,7 +85,7 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
 
 
 # Tests the whole stack works with staus codes ignored
-def test_403_page_check_works_with_ignore_status_code(client, live_server):
+def test_403_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage):
     sleep_time_for_fetch_thread = 3
 
     set_original_response()

changedetectionio/tests/test_ignorewhitespace.py

@@ -49,7 +49,7 @@ def set_original_ignore_response():
 
 
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_check_ignore_whitespace(client, live_server):
+def test_check_ignore_whitespace(client, live_server, measure_memory_usage):
     sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up

changedetectionio/tests/test_import.py

@@ -8,10 +8,10 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-def test_import(client, live_server):
+def test_import(client, live_server, measure_memory_usage):
     # Give the endpoint time to spin up
     wait_for_all_checks(client)
 
@@ -34,7 +34,7 @@ https://example.com tag1, other tag"""
     res = client.get( url_for("index"))
     res = client.get( url_for("index"))
 
-def xtest_import_skip_url(client, live_server):
+def xtest_import_skip_url(client, live_server, measure_memory_usage):
 
 
     # Give the endpoint time to spin up
@@ -57,7 +57,7 @@ def xtest_import_skip_url(client, live_server):
     # Clear flask alerts
     res = client.get( url_for("index"))
 
-def test_import_distillio(client, live_server):
+def test_import_distillio(client, live_server, measure_memory_usage):
 
     distill_data='''
 {
@@ -123,7 +123,7 @@ def test_import_distillio(client, live_server):
     # Clear flask alerts
     res = client.get(url_for("index"))
 
-def test_import_custom_xlsx(client, live_server):
+def test_import_custom_xlsx(client, live_server, measure_memory_usage):
     """Test can upload a excel spreadsheet and the watches are created correctly"""
 
     #live_server_setup(live_server)
@@ -172,7 +172,7 @@ def test_import_custom_xlsx(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_import_watchete_xlsx(client, live_server):
+def test_import_watchete_xlsx(client, live_server, measure_memory_usage):
     """Test can upload a excel spreadsheet and the watches are created correctly"""
 
     #live_server_setup(live_server)

changedetectionio/tests/test_jinja2.py

@@ -5,11 +5,11 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_jinja2_in_url_query(client, live_server):
+def test_jinja2_in_url_query(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     # Add our URL to the import page
@@ -34,7 +34,7 @@ def test_jinja2_in_url_query(client, live_server):
     assert b'date=2' in res.data
 
 # https://techtonics.medium.com/secure-templating-with-jinja2-understanding-ssti-and-jinja2-sandbox-environment-b956edd60456
-def test_jinja2_security_url_query(client, live_server):
+def test_jinja2_security_url_query(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     # Add our URL to the import page

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -41,19 +41,26 @@ and it can also be repeated
     from .. import html_tools
 
     # See that we can find the second <script> one, which is not broken, and matches our filter
-    text = html_tools.extract_json_as_string(content, "json:$.offers.price")
-    assert text == "23.5"
+    text = html_tools.extract_json_as_string(content, "json:$.offers.priceCurrency")
+    assert text == '"AUD"'
+
+    text = html_tools.extract_json_as_string('{"id":5}', "json:$.id")
+    assert text == "5"
 
     # also check for jq
     if jq_support:
-        text = html_tools.extract_json_as_string(content, "jq:.offers.price")
-        assert text == "23.5"
+        text = html_tools.extract_json_as_string(content, "jq:.offers.priceCurrency")
+        assert text == '"AUD"'
 
         text = html_tools.extract_json_as_string('{"id":5}', "jq:.id")
         assert text == "5"
 
-    text = html_tools.extract_json_as_string('{"id":5}', "json:$.id")
-    assert text == "5"
+        text = html_tools.extract_json_as_string(content, "jqraw:.offers.priceCurrency")
+        assert text == "AUD"
+
+        text = html_tools.extract_json_as_string('{"id":5}', "jqraw:.id")
+        assert text == "5"
+
 
     # When nothing at all is found, it should throw JSONNOTFound
     # Which is caught and shown to the user in the watch-overview table
@@ -64,6 +71,9 @@ and it can also be repeated
         with pytest.raises(html_tools.JSONNotFound) as e_info:
             html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jq:.id")
 
+        with pytest.raises(html_tools.JSONNotFound) as e_info:
+            html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jqraw:.id")
+
 
 def test_unittest_inline_extract_body():
     content = """
@@ -191,7 +201,7 @@ def set_modified_response():
 
     return None
 
-def test_check_json_without_filter(client, live_server):
+def test_check_json_without_filter(client, live_server, measure_memory_usage):
     # Request a JSON document from a application/json source containing HTML
     # and be sure it doesn't get chewed up by instriptis
     set_json_response_with_html()
@@ -284,13 +294,17 @@ def check_json_filter(json_filter, client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_check_jsonpath_filter(client, live_server):
+def test_check_jsonpath_filter(client, live_server, measure_memory_usage):
     check_json_filter('json:boss.name', client, live_server)
 
-def test_check_jq_filter(client, live_server):
+def test_check_jq_filter(client, live_server, measure_memory_usage):
     if jq_support:
         check_json_filter('jq:.boss.name', client, live_server)
 
+def test_check_jqraw_filter(client, live_server, measure_memory_usage):
+    if jq_support:
+        check_json_filter('jqraw:.boss.name', client, live_server)
+
 def check_json_filter_bool_val(json_filter, client, live_server):
     set_original_response()
 
@@ -338,10 +352,14 @@ def check_json_filter_bool_val(json_filter, client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_check_jsonpath_filter_bool_val(client, live_server):
+def test_check_jsonpath_filter_bool_val(client, live_server, measure_memory_usage):
     check_json_filter_bool_val("json:$['available']", client, live_server)
 
-def test_check_jq_filter_bool_val(client, live_server):
+def test_check_jq_filter_bool_val(client, live_server, measure_memory_usage):
+    if jq_support:
+        check_json_filter_bool_val("jq:.available", client, live_server)
+
+def test_check_jqraw_filter_bool_val(client, live_server, measure_memory_usage):
     if jq_support:
         check_json_filter_bool_val("jq:.available", client, live_server)
 
@@ -412,7 +430,7 @@ def check_json_ext_filter(json_filter, client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_ignore_json_order(client, live_server):
+def test_ignore_json_order(client, live_server, measure_memory_usage):
     # A change in order shouldn't trigger a notification
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -454,7 +472,7 @@ def test_ignore_json_order(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_correct_header_detect(client, live_server):
+def test_correct_header_detect(client, live_server, measure_memory_usage):
     # Like in https://github.com/dgtlmoon/changedetection.io/pull/1593
     # Specify extra html that JSON is sometimes wrapped in - when using SockpuppetBrowser / Puppeteer / Playwrightetc
     with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -486,9 +504,13 @@ def test_correct_header_detect(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_check_jsonpath_ext_filter(client, live_server):
+def test_check_jsonpath_ext_filter(client, live_server, measure_memory_usage):
     check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server)
 
-def test_check_jq_ext_filter(client, live_server):
+def test_check_jq_ext_filter(client, live_server, measure_memory_usage):
     if jq_support:
-        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
+        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
+
+def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage):
+    if jq_support:
+        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)

changedetectionio/tests/test_nonrenderable_pages.py

@@ -22,7 +22,7 @@ def set_nonrenderable_response():
 
     return None
 
-def test_check_basic_change_detection_functionality(client, live_server):
+def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
     set_original_response()
     live_server_setup(live_server)
 

changedetectionio/tests/test_notification.py

@@ -21,7 +21,7 @@ def test_setup(live_server):
 
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
-def test_check_notification(client, live_server):
+def test_check_notification(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     set_original_response()
 
@@ -234,7 +234,7 @@ def test_check_notification(client, live_server):
         follow_redirects=True
     )
 
-def test_notification_validation(client, live_server):
+def test_notification_validation(client, live_server, measure_memory_usage):
 
     time.sleep(1)
 
@@ -273,7 +273,7 @@ def test_notification_validation(client, live_server):
 
 
 
-def test_notification_custom_endpoint_and_jinja2(client, live_server):
+def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     # test_endpoint - that sends the contents of a file

changedetectionio/tests/test_notification_errors.py

@@ -4,14 +4,11 @@ from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 import logging
 
-def test_check_notification_error_handling(client, live_server):
+def test_check_notification_error_handling(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
     set_original_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Set a URL and fetch it, then set a notification URL which is going to give errors
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(

changedetectionio/tests/test_obfuscations.py

@@ -18,7 +18,7 @@ def set_original_ignore_response():
         f.write(test_return_data)
 
 
-def test_obfuscations(client, live_server):
+def test_obfuscations(client, live_server, measure_memory_usage):
     set_original_ignore_response()
     live_server_setup(live_server)
     time.sleep(1)

changedetectionio/tests/test_pdf.py

@@ -6,7 +6,7 @@ from .util import set_original_response, set_modified_response, live_server_setu
 
 
 # `subtractive_selectors` should still work in `source:` type requests
-def test_fetch_pdf(client, live_server):
+def test_fetch_pdf(client, live_server, measure_memory_usage):
     import shutil
     shutil.copy("tests/test.pdf", "test-datastore/endpoint-test.pdf")
 

changedetectionio/tests/test_request.py

@@ -9,7 +9,7 @@ def test_setup(live_server):
 
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
-def test_headers_in_request(client, live_server):
+def test_headers_in_request(client, live_server, measure_memory_usage):
     #ve_server_setup(live_server)
     # Add our URL to the import page
     test_url = url_for('test_headers', _external=True)
@@ -84,7 +84,7 @@ def test_headers_in_request(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_body_in_request(client, live_server):
+def test_body_in_request(client, live_server, measure_memory_usage):
 
     # Add our URL to the import page
     test_url = url_for('test_body', _external=True)
@@ -177,7 +177,7 @@ def test_body_in_request(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_method_in_request(client, live_server):
+def test_method_in_request(client, live_server, measure_memory_usage):
     # Add our URL to the import page
     test_url = url_for('test_method', _external=True)
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):
@@ -253,7 +253,63 @@ def test_method_in_request(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_headers_textfile_in_request(client, live_server):
+# Re #2408 - user-agent override test, also should handle case-insensitive header deduplication
+def test_ua_global_override(client, live_server, measure_memory_usage):
+    # live_server_setup(live_server)
+    test_url = url_for('test_headers', _external=True)
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "application-fetch_backend": "html_requests",
+            "application-minutes_between_check": 180,
+            "requests-default_ua-html_requests": "html-requests-user-agent"
+        },
+        follow_redirects=True
+    )
+    assert b'Settings updated' in res.data
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    wait_for_all_checks(client)
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"html-requests-user-agent" in res.data
+    # default user-agent should have shown by now
+    # now add a custom one in the headers
+
+
+    # Add some headers to a request
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "url": test_url,
+            "tags": "testtag",
+            "fetch_backend": 'html_requests',
+            # Important - also test case-insensitive
+            "headers": "User-AGent: agent-from-watch"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+    wait_for_all_checks(client)
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"agent-from-watch" in res.data
+    assert b"html-requests-user-agent" not in res.data
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     # Add our URL to the import page
 
@@ -333,7 +389,7 @@ def test_headers_textfile_in_request(client, live_server):
     # Not needed anymore
     os.unlink('test-datastore/headers.txt')
     os.unlink('test-datastore/headers-testtag.txt')
-    os.unlink('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt')
+
     # The service should echo back the request verb
     res = client.get(
         url_for("preview_page", uuid="first"),

changedetectionio/tests/test_rss.py

@@ -49,10 +49,10 @@ def set_original_cdata_xml():
         f.write(test_return_data)
 
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-def test_rss_and_token(client, live_server):
+def test_rss_and_token(client, live_server, measure_memory_usage):
     #    live_server_setup(live_server)
 
     set_original_response()
@@ -69,6 +69,7 @@ def test_rss_and_token(client, live_server):
 
     wait_for_all_checks(client)
     set_modified_response()
+    time.sleep(1)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -87,9 +88,9 @@ def test_rss_and_token(client, live_server):
     assert b"Access denied, bad token" not in res.data
     assert b"Random content" in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
-def test_basic_cdata_rss_markup(client, live_server):
+def test_basic_cdata_rss_markup(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     set_original_cdata_xml()
@@ -117,7 +118,7 @@ def test_basic_cdata_rss_markup(client, live_server):
     assert b'The days of Terminator' in res.data
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
-def test_rss_xpath_filtering(client, live_server):
+def test_rss_xpath_filtering(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     set_original_cdata_xml()

changedetectionio/tests/test_search.py

@@ -5,7 +5,7 @@ import time
 def test_setup(live_server):
     live_server_setup(live_server)
 
-def test_basic_search(client, live_server):
+def test_basic_search(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     urls = ['https://localhost:12300?first-result=1',
@@ -38,7 +38,7 @@ def test_basic_search(client, live_server):
     assert urls[1].encode('utf-8') not in res.data
 
 
-def test_search_in_tag_limit(client, live_server):
+def test_search_in_tag_limit(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     urls = ['https://localhost:12300?first-result=1 tag-one',

changedetectionio/tests/test_security.py

@@ -2,10 +2,10 @@ from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 import time
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-def test_bad_access(client, live_server):
+def test_bad_access(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     res = client.post(
         url_for("import_page"),
@@ -67,7 +67,7 @@ def test_bad_access(client, live_server):
 
     assert b'file:// type access is denied for security reasons.' in res.data
 
-def test_xss(client, live_server):
+def test_xss(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
     from changedetectionio.notification import (
         default_notification_format

changedetectionio/tests/test_share_watch.py

@@ -9,7 +9,7 @@ import re
 sleep_time_for_fetch_thread = 3
 
 
-def test_share_watch(client, live_server):
+def test_share_watch(client, live_server, measure_memory_usage):
     set_original_response()
     live_server_setup(live_server)
 

changedetectionio/tests/test_source.py

@@ -10,7 +10,7 @@ sleep_time_for_fetch_thread = 3
 def test_setup(live_server):
     live_server_setup(live_server)
 
-def test_check_basic_change_detection_functionality_source(client, live_server):
+def test_check_basic_change_detection_functionality_source(client, live_server, measure_memory_usage):
     set_original_response()
     test_url = 'source:'+url_for('test_endpoint', _external=True)
     # Add our URL to the import page
@@ -58,7 +58,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
 
 
 # `subtractive_selectors` should still work in `source:` type requests
-def test_check_ignore_elements(client, live_server):
+def test_check_ignore_elements(client, live_server, measure_memory_usage):
     set_original_response()
     time.sleep(1)
     test_url = 'source:'+url_for('test_endpoint', _external=True)

changedetectionio/tests/test_trigger.py

@@ -55,7 +55,7 @@ def set_modified_with_trigger_text_response():
         f.write(test_return_data)
 
 
-def test_trigger_functionality(client, live_server):
+def test_trigger_functionality(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
 

changedetectionio/tests/test_trigger_regex.py

@@ -22,7 +22,7 @@ def set_original_ignore_response():
 
 
 
-def test_trigger_regex_functionality(client, live_server):
+def test_trigger_regex_functionality(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
 

changedetectionio/tests/test_trigger_regex_with_filter.py

@@ -22,7 +22,7 @@ def set_original_ignore_response():
 
 
 
-def test_trigger_regex_functionality_with_filter(client, live_server):
+def test_trigger_regex_functionality_with_filter(client, live_server, measure_memory_usage):
 
     live_server_setup(live_server)
     sleep_time_for_fetch_thread = 3

changedetectionio/tests/test_unique_lines.py

@@ -66,10 +66,10 @@ def set_modified_with_trigger_text_response():
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-def test_unique_lines_functionality(client, live_server):
+def test_unique_lines_functionality(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
 
@@ -118,7 +118,7 @@ def test_unique_lines_functionality(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_sort_lines_functionality(client, live_server):
+def test_sort_lines_functionality(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     set_modified_swapped_lines_with_extra_text_for_sorting()

changedetectionio/tests/test_watch_fields_storage.py

@@ -4,7 +4,7 @@ from urllib.request import urlopen
 from . util import set_original_response, set_modified_response, live_server_setup
 
 
-def test_check_watch_field_storage(client, live_server):
+def test_check_watch_field_storage(client, live_server, measure_memory_usage):
     set_original_response()
     live_server_setup(live_server)
 

changedetectionio/tests/test_xpath_selector.py

@@ -49,7 +49,7 @@ def set_modified_response():
 
 
 # Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613
-def test_check_xpath_filter_utf8(client, live_server):
+def test_check_xpath_filter_utf8(client, live_server, measure_memory_usage):
     filter = '//item/*[self::description]'
 
     d = '''<?xml version="1.0" encoding="UTF-8"?>
@@ -105,7 +105,7 @@ def test_check_xpath_filter_utf8(client, live_server):
 
 
 # Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613
-def test_check_xpath_text_function_utf8(client, live_server):
+def test_check_xpath_text_function_utf8(client, live_server, measure_memory_usage):
     filter = '//item/title/text()'
 
     d = '''<?xml version="1.0" encoding="UTF-8"?>
@@ -168,7 +168,7 @@ def test_check_xpath_text_function_utf8(client, live_server):
     assert b'Deleted' in res.data
 
 
-def test_check_markup_xpath_filter_restriction(client, live_server):
+def test_check_markup_xpath_filter_restriction(client, live_server, measure_memory_usage):
     xpath_filter = "//*[contains(@class, 'sametext')]"
 
     set_original_response()
@@ -214,7 +214,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     assert b'Deleted' in res.data
 
 
-def test_xpath_validation(client, live_server):
+def test_xpath_validation(client, live_server, measure_memory_usage):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -235,7 +235,7 @@ def test_xpath_validation(client, live_server):
     assert b'Deleted' in res.data
 
 
-def test_xpath23_prefix_validation(client, live_server):
+def test_xpath23_prefix_validation(client, live_server, measure_memory_usage):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -255,7 +255,7 @@ def test_xpath23_prefix_validation(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_xpath1_lxml(client, live_server):
+def test_xpath1_lxml(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
     d = '''<?xml version="1.0" encoding="UTF-8"?>
@@ -319,7 +319,7 @@ def test_xpath1_lxml(client, live_server):
     #####
 
 
-def test_xpath1_validation(client, live_server):
+def test_xpath1_validation(client, live_server, measure_memory_usage):
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -341,7 +341,7 @@ def test_xpath1_validation(client, live_server):
 
 
 # actually only really used by the distll.io importer, but could be handy too
-def test_check_with_prefix_include_filters(client, live_server):
+def test_check_with_prefix_include_filters(client, live_server, measure_memory_usage):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
@@ -378,7 +378,7 @@ def test_check_with_prefix_include_filters(client, live_server):
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
 
-def test_various_rules(client, live_server):
+def test_various_rules(client, live_server, measure_memory_usage):
     # Just check these don't error
     # live_server_setup(live_server)
     with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -426,7 +426,7 @@ def test_various_rules(client, live_server):
     assert b'Deleted' in res.data
 
 
-def test_xpath_20(client, live_server):
+def test_xpath_20(client, live_server, measure_memory_usage):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("import_page"),
@@ -463,7 +463,7 @@ def test_xpath_20(client, live_server):
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
 
-def test_xpath_20_function_count(client, live_server):
+def test_xpath_20_function_count(client, live_server, measure_memory_usage):
     set_original_response()
 
     # Add our URL to the import page
@@ -499,7 +499,7 @@ def test_xpath_20_function_count(client, live_server):
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
 
-def test_xpath_20_function_count2(client, live_server):
+def test_xpath_20_function_count2(client, live_server, measure_memory_usage):
     set_original_response()
 
     # Add our URL to the import page
@@ -535,7 +535,7 @@ def test_xpath_20_function_count2(client, live_server):
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
 
-def test_xpath_20_function_string_join_matches(client, live_server):
+def test_xpath_20_function_string_join_matches(client, live_server, measure_memory_usage):
     set_original_response()
 
     # Add our URL to the import page

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -4,12 +4,12 @@ import os
 from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
 
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
-def test_visual_selector_content_ready(client, live_server):
+def test_visual_selector_content_ready(client, live_server, measure_memory_usage):
 
     import os
     import json
@@ -79,7 +79,7 @@ def test_visual_selector_content_ready(client, live_server):
         follow_redirects=True
     )
 
-def test_basic_browserstep(client, live_server):
+def test_basic_browserstep(client, live_server, measure_memory_usage):
 
     #live_server_setup(live_server)
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"

changedetectionio/update_worker.py

@@ -1,11 +1,12 @@
-import os
-import threading
-import queue
-import time
 from . import content_fetchers
-from changedetectionio import html_tools
-from .processors.text_json_diff import FilterNotFoundInResponse
 from .processors.restock_diff import UnableToExtractRestockData
+from .processors.text_json_diff import FilterNotFoundInResponse
+from changedetectionio import html_tools
+from copy import deepcopy
+import os
+import queue
+import threading
+import time
 
 # A single update worker
 #
@@ -245,14 +246,18 @@ class update_worker(threading.Thread):
                     contents = b''
                     process_changedetection_results = True
                     update_obj = {}
-                    logger.info(f"Processing watch UUID {uuid} "
-                            f"Priority {queued_item_data.priority} "
-                            f"URL {self.datastore.data['watching'][uuid]['url']}")
+
+                    # Clear last errors (move to preflight func?)
+                    self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
+
+                    watch = self.datastore.data['watching'].get(uuid)
+
+                    logger.info(f"Processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
                     now = time.time()
 
                     try:
                         # Processor is what we are using for detecting the "Change"
-                        processor = self.datastore.data['watching'][uuid].get('processor', 'text_json_diff')
+                        processor = watch.get('processor', 'text_json_diff')
                         # if system...
 
                         # Abort processing when the content was the same as the last fetch
@@ -272,14 +277,12 @@ class update_worker(threading.Thread):
                                                                                watch_uuid=uuid
                                                                                )
 
-                        # Clear last errors (move to preflight func?)
-                        self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
-
                         update_handler.call_browser()
 
-                        changed_detected, update_obj, contents = update_handler.run_changedetection(uuid,
-                                                                                    skip_when_checksum_same=skip_when_same_checksum,
-                                                                                    )
+                        changed_detected, update_obj, contents = update_handler.run_changedetection(
+                            watch=watch,
+                            skip_when_checksum_same=skip_when_same_checksum,
+                        )
 
                         # Re #342
                         # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
@@ -309,7 +312,11 @@ class update_worker(threading.Thread):
                         })
 
                         if e.screenshot:
-                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot)
+                            watch.save_screenshot(screenshot=e.screenshot, as_error=True)
+
+                        if e.xpath_data:
+                            watch.save_xpath_data(data=e.xpath_data)
+                            
                         process_changedetection_results = False
 
                     except content_fetchers.exceptions.Non200ErrorCodeReceived as e:
@@ -325,11 +332,11 @@ class update_worker(threading.Thread):
                             err_text = "Error - Request returned a HTTP error code {}".format(str(e.status_code))
 
                         if e.screenshot:
-                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
+                            watch.save_screenshot(screenshot=e.screenshot, as_error=True)
                         if e.xpath_data:
-                            self.datastore.save_xpath_data(watch_uuid=uuid, data=e.xpath_data, as_error=True)
+                            watch.save_xpath_data(data=e.xpath_data, as_error=True)
                         if e.page_text:
-                            self.datastore.save_error_text(watch_uuid=uuid, contents=e.page_text)
+                            watch.save_error_text(contents=e.page_text)
 
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
                         process_changedetection_results = False
@@ -341,16 +348,23 @@ class update_worker(threading.Thread):
                         err_text = "Warning, no filters were found, no change detection ran - Did the page change layout? update your Visual Filter if necessary."
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
 
+                        # Filter wasnt found, but we should still update the visual selector so that they can have a chance to set it up again
+                        if e.screenshot:
+                            watch.save_screenshot(screenshot=e.screenshot)
+
+                        if e.xpath_data:
+                            watch.save_xpath_data(data=e.xpath_data)
+
                         # Only when enabled, send the notification
-                        if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
-                            c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
+                        if watch.get('filter_failure_notification_send', False):
+                            c = watch.get('consecutive_filter_failures', 5)
                             c += 1
                             # Send notification if we reached the threshold?
                             threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
                                                                                            0)
-                            logger.error(f"Filter for {uuid} not found, consecutive_filter_failures: {c}")
+                            logger.warning(f"Filter for {uuid} not found, consecutive_filter_failures: {c}")
                             if threshold > 0 and c >= threshold:
-                                if not self.datastore.data['watching'][uuid].get('notification_muted'):
+                                if not watch.get('notification_muted'):
                                     self.send_filter_failure_notification(uuid)
                                 c = 0
 
@@ -362,7 +376,6 @@ class update_worker(threading.Thread):
                         # Yes fine, so nothing todo, don't continue to process.
                         process_changedetection_results = False
                         changed_detected = False
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False})
                     except content_fetchers.exceptions.BrowserConnectError as e:
                         self.datastore.update_watch(uuid=uuid,
                                                     update_obj={'last_error': e.msg
@@ -401,15 +414,15 @@ class update_worker(threading.Thread):
                                                                 }
                                                     )
 
-                        if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
-                            c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
+                        if watch.get('filter_failure_notification_send', False):
+                            c = watch.get('consecutive_filter_failures', 5)
                             c += 1
                             # Send notification if we reached the threshold?
                             threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
                                                                                            0)
                             logger.error(f"Step for {uuid} not found, consecutive_filter_failures: {c}")
                             if threshold > 0 and c >= threshold:
-                                if not self.datastore.data['watching'][uuid].get('notification_muted'):
+                                if not watch.get('notification_muted'):
                                     self.send_step_failure_notification(watch_uuid=uuid, step_n=e.step_n)
                                 c = 0
 
@@ -431,7 +444,7 @@ class update_worker(threading.Thread):
                     except content_fetchers.exceptions.JSActionExceptions as e:
                         err_text = "Error running JS Actions - Page request - "+e.message
                         if e.screenshot:
-                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
+                            watch.save_screenshot(screenshot=e.screenshot, as_error=True)
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
                         process_changedetection_results = False
@@ -441,7 +454,7 @@ class update_worker(threading.Thread):
                             err_text = "{} - {}".format(err_text, e.message)
 
                         if e.screenshot:
-                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
+                            watch.save_screenshot(screenshot=e.screenshot, as_error=True)
 
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code,
@@ -465,8 +478,6 @@ class update_worker(threading.Thread):
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Exception: " + str(e)})
                         # Other serious error
                         process_changedetection_results = False
-#                        import traceback
-#                        print(traceback.format_exc())
 
                     else:
                         # Crash protection, the watch entry could have been removed by this point (during a slow chrome fetch etc)
@@ -474,7 +485,7 @@ class update_worker(threading.Thread):
                             continue
 
                         # Mark that we never had any failures
-                        if not self.datastore.data['watching'][uuid].get('ignore_status_codes'):
+                        if not watch.get('ignore_status_codes'):
                             update_obj['consecutive_filter_failures'] = 0
 
                         # Everything ran OK, clean off any previous error
@@ -482,25 +493,48 @@ class update_worker(threading.Thread):
 
                         self.cleanup_error_artifacts(uuid)
 
+                    if not self.datastore.data['watching'].get(uuid):
+                        continue
                     #
                     # Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
                     if process_changedetection_results:
+                        # Always save the screenshot if it's available
+
+                        if update_handler.screenshot:
+                            watch.save_screenshot(screenshot=update_handler.screenshot)
+
+                        if update_handler.xpath_data:
+                            watch.save_xpath_data(data=update_handler.xpath_data)
+
                         try:
-                            watch = self.datastore.data['watching'].get(uuid)
                             self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
 
                             # Also save the snapshot on the first time checked
-                            if changed_detected or not watch['last_checked']:
+                            if changed_detected or not watch.get('last_checked'):
+                                timestamp = round(time.time())
+
+                                # Small hack so that we sleep just enough to allow 1 second  between history snapshots
+                                # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
+
+                                if watch.newest_history_key and int(timestamp) == int(watch.newest_history_key):
+                                    logger.warning(
+                                        f"Timestamp {timestamp} already exists, waiting 1 seconds so we have a unique key in history.txt")
+                                    timestamp = str(int(timestamp) + 1)
+                                    time.sleep(1)
+
                                 watch.save_history_text(contents=contents,
-                                                        timestamp=str(round(time.time())),
+                                                        timestamp=timestamp,
                                                         snapshot_id=update_obj.get('previous_md5', 'none'))
 
+                                if update_handler.fetcher.content:
+                                    watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=timestamp)
+
                             # A change was detected
                             if changed_detected:
                                 # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
                                 if watch.history_n >= 2:
                                     logger.info(f"Change detected in UUID {uuid} - {watch['url']}")
-                                    if not self.datastore.data['watching'][uuid].get('notification_muted'):
+                                    if not watch.get('notification_muted'):
                                         self.send_content_changed_notification(watch_uuid=uuid)
                                 else:
                                     logger.info(f"Change triggered in UUID {uuid} due to first history saving (no notifications sent) - {watch['url']}")
@@ -511,29 +545,23 @@ class update_worker(threading.Thread):
                             logger.critical(str(e))
                             self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
 
-                    if self.datastore.data['watching'].get(uuid):
-                        # Always record that we atleast tried
-                        count = self.datastore.data['watching'][uuid].get('check_count', 0) + 1
 
-                        # Record the 'server' header reply, can be used for actions in the future like cloudflare/akamai workarounds
-                        try:
-                            server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]
-                            self.datastore.update_watch(uuid=uuid,
-                                                        update_obj={'remote_server_reply': server_header}
-                                                        )
-                        except Exception as e:
-                            pass
+                    # Always record that we atleast tried
+                    count = watch.get('check_count', 0) + 1
 
-                        self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
-                                                                           'last_checked': round(time.time()),
-                                                                           'check_count': count
-                                                                           })
+                    # Record the 'server' header reply, can be used for actions in the future like cloudflare/akamai workarounds
+                    try:
+                        server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]
+                        self.datastore.update_watch(uuid=uuid,
+                                                    update_obj={'remote_server_reply': server_header}
+                                                    )
+                    except Exception as e:
+                        pass
 
-                        # Always save the screenshot if it's available
-                        if update_handler.screenshot:
-                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=update_handler.screenshot)
-                        if update_handler.xpath_data:
-                            self.datastore.save_xpath_data(watch_uuid=uuid, data=update_handler.xpath_data)
+                    self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
+                                                                       'last_checked': round(time.time()),
+                                                                       'check_count': count
+                                                                       })
 
 
                 self.current_uuid = None  # Done

docker-compose.yml

@@ -68,9 +68,10 @@ services:
      
      # If WEBDRIVER or PLAYWRIGHT are enabled, changedetection container depends on that
      # and must wait before starting (substitute "browser-chrome" with "playwright-chrome" if last one is used)
-#    depends_on:
-#        browser-chrome:
-#            condition: service_started
+#      depends_on:
+#          playwright-chrome:
+#              condition: service_started
+
 
      # Used for fetching pages via Playwright+Chrome where you need Javascript support.
      # RECOMMENDED FOR FETCHING PAGES WITH CHROME

requirements.txt

@@ -1,7 +1,7 @@
 # Used by Pyppeteer
 pyee
 
-eventlet==0.35.2 # related to dnspython fixes
+eventlet>=0.36.1 # fixes SSL error on Python 3.12
 feedgen~=0.9
 flask-compress
 # 0.6.3 included compatibility fix for werkzeug 3.x (2.x had deprecation of url handlers)
@@ -23,15 +23,13 @@ validators~=0.21
 brotli~=1.0
 requests[socks]
 
-urllib3==1.26.18
+urllib3==1.26.19
 chardet>2.3.0
 
 wtforms~=3.0
 jsonpath-ng~=1.5.3
 
-# Pinned: module 'eventlet.green.select' has no attribute 'epoll'
-# https://github.com/eventlet/eventlet/issues/805#issuecomment-1640463482
-dnspython==2.3.0 # related to eventlet fixes
+dnspython==2.6.1 # related to eventlet fixes
 
 # jq not available on Windows so must be installed manually
 
@@ -43,10 +41,8 @@ apprise~=1.8.0
 # use v1.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
 paho-mqtt>=1.6.1,<2.0.0
 
-# This mainly affects some ARM builds, which unlike the other builds ignores "ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1"
-# so without this pinning, the newer versions on ARM will forcefully try to build rust, which results in "rust compiler not found"
-# (introduced once apprise became a dep)
-cryptography~=3.4
+# Requires extra wheel for rPi
+cryptography~=42.0.8
 
 # Used for CSS filtering
 beautifulsoup4
@@ -86,3 +82,5 @@ pytest-flask ~=1.2
 jsonschema==4.17.3
 
 loguru
+# Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096
+greenlet >= 3.0.3