Fixes for status code and screenshot info

Added `prodotto esaurito` - Italian for out of stock

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: "weekly"
+    groups:
+      all:
+        patterns:
+        - "*"

.github/workflows/codeql-analysis.yml

@@ -34,7 +34,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -45,7 +45,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -59,4 +59,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/containers.yml

@@ -41,7 +41,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python 3.11
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
 

.github/workflows/pypi-release.yml

@@ -9,7 +9,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "3.x"
     - name: Install pypa/build
@@ -21,7 +21,7 @@ jobs:
     - name: Build a binary wheel and a source tarball
       run: python3 -m build
     - name: Store the distribution packages
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: python-package-distributions
         path: dist/
@@ -34,18 +34,18 @@ jobs:
     - build
     steps:
     - name: Download all the dists
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: python-package-distributions
         path: dist/
     - name: Test that the basic pip built package runs without error
       run: |
-        set -e
+        set -ex
         pip3 install dist/changedetection.io*.whl
         changedetection.io -d /tmp -p 10000 &
         sleep 3
-        curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
-        curl http://127.0.0.1:10000/ >/dev/null
+        curl --retry-connrefused --retry 6 http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
+        curl --retry-connrefused --retry 6 http://127.0.0.1:10000/ >/dev/null
         killall changedetection.io
 
 
@@ -64,7 +64,7 @@ jobs:
 
     steps:
     - name: Download all the dists
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: python-package-distributions
         path: dist/

.github/workflows/test-container-build.yml

@@ -26,7 +26,7 @@ jobs:
     steps:
         - uses: actions/checkout@v4
         - name: Set up Python 3.11
-          uses: actions/setup-python@v4
+          uses: actions/setup-python@v5
           with:
             python-version: 3.11
 

.github/workflows/test-only.yml

@@ -11,7 +11,7 @@ jobs:
 
       # Mainly just for link/flake8
       - name: Set up Python 3.11
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.11'
 
@@ -27,18 +27,18 @@ jobs:
         run: |
           
           docker network create changedet-network
-
-          # Selenium+browserless
+          
+          # Selenium and sockpuppetbrowser
           docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
-          docker run --network changedet-network -d --name browserless --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.60-chrome-stable
+          docker run --network changedet-network -d --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
           
           # For accessing custom browser tests
-          docker run --network changedet-network -d --name browserless-custom-url --hostname browserless-custom-url -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm --shm-size="2g"  browserless/chrome:1.60-chrome-stable
+          docker run --network changedet-network -d --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url --rm dgtlmoon/sockpuppetbrowser:latest
 
       - name: Build changedetection.io container for testing
         run: |         
           # Build a changedetection.io container and start testing inside
-          docker build . -t test-changedetectionio
+          docker build --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
           # Debug info
           docker run test-changedetectionio  bash -c 'pip list'
 
@@ -47,42 +47,55 @@ jobs:
           # Debug SMTP server/echo message back server
           docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'python changedetectionio/tests/smtp/smtp-test-server.py' 
 
-      - name: Test built container with pytest
+      - name: Show docker container state and other debug info
+        run: |
+          set -x
+          echo "Running processes in docker..."
+          docker ps
+
+      - name: Test built container with Pytest (generally as requests/plaintext fetching)
         run: |
           # Unit tests
+          echo "run test with unittest"
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
           
           # All tests
+          echo "run test with pytest"
+          # The default pytest logger_level is TRACE
+          # To change logger_level for pytest(test/conftest.py),
+          # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
           docker run --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
 
-      - name: Test built container selenium+browserless/playwright
+      - name: Specific tests in built container for Selenium
         run: |
-          
           # Selenium fetch
           docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
-          
-          # Playwright/Browserless fetch
-          docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
-          
-          # Settings headers playwright tests - Call back in from Browserless, check headers
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+
+      - name: Specific tests in built container for Playwright and SocketPuppetBrowser
+        run: |
+          # Playwright via Sockpuppetbrowser fetch
+          docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
+
+      - name: Specific tests in built container for headers and requests checks with Playwright
+        run: |       
+          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+
+      - name: Specific tests in built container for headers and requests checks with Selenium
+        run: |
           docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'          
-          
-          # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
+      - name: Test built container restock detection via Playwright
+        run: |                            
+          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
 
       - name: Test SMTP notification mime types
         run: |
           # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
           docker run --rm  --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
 
-      - name: Test with puppeteer fetcher and disk cache
-        run: |
-          docker run --rm -e "PUPPETEER_DISK_CACHE=/tmp/data/" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
-          # Browserless would have had -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" added above
-
       - name: Test proxy interaction
         run: |
           cd changedetectionio
@@ -101,10 +114,17 @@ jobs:
           docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
           sleep 3
           # Should return 0 (no error) when grep finds it
-          curl -s http://localhost:5556 |grep -q checkbox-uuid
+          curl --retry-connrefused --retry 6  -s http://localhost:5556 |grep -q checkbox-uuid
           
           # and IPv6
-          curl -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+          curl --retry-connrefused --retry 6  -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+
+          # Check whether TRACE log is enabled.
+          # Also, check whether TRACE is came from STDERR
+          docker logs test-changedetectionio 2>&1 1>/dev/null | grep 'TRACE log is enabled' || exit 1
+          # Check whether DEBUG is came from STDOUT
+          docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
+
           docker kill test-changedetectionio
 
       - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
@@ -118,8 +138,9 @@ jobs:
           sleep 3
           # invert the check (it should be not 0/not running)
           docker ps
-          # check signal catch(STDOUT) log
-          docker logs sig-test | grep 'Shutdown: Got Signal - SIGINT' || exit 1
+          # check signal catch(STDERR) log. Because of
+          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
+          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
           test -z "`docker ps|grep sig-test`"
           if [ $? -ne 0 ]
           then
@@ -139,7 +160,9 @@ jobs:
           sleep 3
           # invert the check (it should be not 0/not running)
           docker ps
-          docker logs sig-test | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
+          # check signal catch(STDERR) log. Because of
+          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
+          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
           test -z "`docker ps|grep sig-test`"
           if [ $? -ne 0 ]
           then

Dockerfile

@@ -58,6 +58,11 @@ COPY changedetectionio /app/changedetectionio
 # Starting wrapper
 COPY changedetection.py /app/changedetection.py
 
+# Github Action test purpose(test-only.yml).
+# On production, it is effectively LOGGER_LEVEL=''.
+ARG LOGGER_LEVEL=''
+ENV LOGGER_LEVEL "$LOGGER_LEVEL"
+
 WORKDIR /app
 CMD ["python", "./changedetection.py", "-d", "/datastore"]
 

README.md

@@ -249,7 +249,7 @@ Supports managing the website watch list [via our API](https://changedetection.i
 Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
 
 
-Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
+Firstly, consider taking out an officially supported [website change detection subscription](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
 
 Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ)
 
@@ -273,3 +273,9 @@ I offer commercial support, this software is depended on by network security, ae
 ## Third-party licenses
 
 changedetectionio.html_tools.elementpath_tostring: Copyright (c), 2018-2021, SISSA (Scuola Internazionale Superiore di Studi Avanzati), Licensed under [MIT license](https://github.com/sissaschool/elementpath/blob/master/LICENSE)
+
+## Contributors
+
+Recognition of fantastic contributors to the project
+
+- Constantin Hong https://github.com/Constantin1489

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.45.12'
+__version__ = '0.45.14'
 
 from distutils.util import strtobool
 from json.decoder import JSONDecodeError
@@ -17,6 +17,7 @@ import sys
 
 from changedetectionio import store
 from changedetectionio.flask_app import changedetection_app
+from loguru import logger
 
 
 # Only global so we can access it in the signal handler
@@ -28,9 +29,9 @@ def sigshutdown_handler(_signo, _stack_frame):
     global app
     global datastore
     name = signal.Signals(_signo).name
-    print(f'Shutdown: Got Signal - {name} ({_signo}), Saving DB to disk and calling shutdown')
+    logger.critical(f'Shutdown: Got Signal - {name} ({_signo}), Saving DB to disk and calling shutdown')
     datastore.sync_to_json()
-    print(f'Sync JSON to disk complete.')
+    logger.success('Sync JSON to disk complete.')
     # This will throw a SystemExit exception, because eventlet.wsgi.server doesn't know how to deal with it.
     # Solution: move to gevent or other server in the future (#2014)
     datastore.stop_thread = True
@@ -57,13 +58,22 @@ def main():
         datastore_path = os.path.join(os.getcwd(), "../datastore")
 
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:", "port")
+        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:l:", "port")
     except getopt.GetoptError:
-        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
+        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path] -l [debug level - TRACE, DEBUG(default), INFO, SUCCESS, WARNING, ERROR, CRITICAL]')
         sys.exit(2)
 
     create_datastore_dir = False
 
+    # Set a default logger level
+    logger_level = 'DEBUG'
+    # Set a logger level via shell env variable
+    # Used: Dockerfile for CICD
+    # To set logger level for pytest, see the app function in tests/conftest.py
+    if os.getenv("LOGGER_LEVEL"):
+        level = os.getenv("LOGGER_LEVEL")
+        logger_level = int(level) if level.isdigit() else level.upper()
+
     for opt, arg in opts:
         if opt == '-s':
             ssl_mode = True
@@ -78,7 +88,7 @@ def main():
             datastore_path = arg
 
         if opt == '-6':
-            print ("Enabling IPv6 listen support")
+            logger.success("Enabling IPv6 listen support")
             ipv6_enabled = True
 
         # Cleanup (remove text files that arent in the index)
@@ -89,6 +99,25 @@ def main():
         if opt == '-C':
             create_datastore_dir = True
 
+        if opt == '-l':
+            logger_level = int(arg) if arg.isdigit() else arg.upper()
+
+    # Without this, a logger will be duplicated
+    logger.remove()
+    try:
+        log_level_for_stdout = { 'DEBUG', 'SUCCESS' }
+        logger.configure(handlers=[
+            {"sink": sys.stdout, "level": logger_level,
+             "filter" : lambda record: record['level'].name in log_level_for_stdout},
+            {"sink": sys.stderr, "level": logger_level,
+             "filter": lambda record: record['level'].name not in log_level_for_stdout},
+            ])
+    # Catch negative number or wrong log level name
+    except ValueError:
+        print("Available log level names: TRACE, DEBUG(default), INFO, SUCCESS,"
+              " WARNING, ERROR, CRITICAL")
+        sys.exit(2)
+
     # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
     app_config = {'datastore_path': datastore_path}
 
@@ -96,17 +125,19 @@ def main():
         if create_datastore_dir:
             os.mkdir(app_config['datastore_path'])
         else:
-            print(
-                "ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists or specify a directory with the -d option.\n"
-                "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
+            logger.critical(
+                f"ERROR: Directory path for the datastore '{app_config['datastore_path']}'"
+                f" does not exist, cannot start, please make sure the"
+                f" directory exists or specify a directory with the -d option.\n"
+                f"Or use the -C parameter to create the directory.")
             sys.exit(2)
 
     try:
         datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
     except JSONDecodeError as e:
         # Dont' start if the JSON DB looks corrupt
-        print ("ERROR: JSON DB or Proxy List JSON at '{}' appears to be corrupt, aborting".format(app_config['datastore_path']))
-        print(str(e))
+        logger.critical(f"ERROR: JSON DB or Proxy List JSON at '{app_config['datastore_path']}' appears to be corrupt, aborting.")
+        logger.critical(str(e))
         return
 
     app = changedetection_app(app_config, datastore)
@@ -145,7 +176,7 @@ def main():
     #         proxy_set_header X-Forwarded-Prefix /app;
 
     if os.getenv('USE_X_SETTINGS'):
-        print ("USE_X_SETTINGS is ENABLED\n")
+        logger.info("USE_X_SETTINGS is ENABLED")
         from werkzeug.middleware.proxy_fix import ProxyFix
         app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
 

changedetectionio/api/api_v1.py

@@ -133,6 +133,7 @@ class WatchHistory(Resource):
 
     # Get a list of available history for a watch by UUID
     # curl http://localhost:5000/api/v1/watch/<string:uuid>/history
+    @auth.check_token
     def get(self, uuid):
         """
         @api {get} /api/v1/watch/<string:uuid>/history Get a list of all historical snapshots available for a watch

changedetectionio/blueprint/browser_steps/__init__.py

@@ -4,30 +4,21 @@
 # Why?
 # `browsersteps_playwright_browser_interface.chromium.connect_over_cdp()` will only run once without async()
 # - this flask app is not async()
-# - browserless has a single timeout/keepalive which applies to the session made at .connect_over_cdp()
+# - A single timeout/keepalive which applies to the session made at .connect_over_cdp()
 #
 # So it means that we must unfortunately for now just keep a single timer since .connect_over_cdp() was run
 # and know when that reaches timeout/keepalive :( when that time is up, restart the connection and tell the user
 # that their time is up, insert another coin. (reload)
 #
-# Bigger picture
-# - It's horrible that we have this click+wait deal, some nice socket.io solution using something similar
-# to what the browserless debug UI already gives us would be smarter..
 #
-# OR
-# - Some API call that should be hacked into browserless or playwright that we can "/api/bump-keepalive/{session_id}/60"
-# So we can tell it that we need more time (run this on each action)
-#
-# OR
-# - use multiprocessing to bump this over to its own process and add some transport layer (queue/pipes)
 
 from distutils.util import strtobool
 from flask import Blueprint, request, make_response
-import logging
 import os
 
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.flask_app import login_optionally_required
+from loguru import logger
 
 browsersteps_sessions = {}
 io_interface_context = None
@@ -88,7 +79,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 if parsed.password:
                     proxy['password'] = parsed.password
 
-                print("Browser Steps: UUID {} selected proxy {}".format(watch_uuid, proxy_url))
+                logger.debug(f"Browser Steps: UUID {watch_uuid} selected proxy {proxy_url}")
 
         # Tell Playwright to connect to Chrome and setup a new session via our stepper interface
         browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui(
@@ -115,10 +106,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         if not watch_uuid:
             return make_response('No Watch UUID specified', 500)
 
-        print("Starting connection with playwright")
-        logging.debug("browser_steps.py connecting")
+        logger.debug("Starting connection with playwright")
+        logger.debug("browser_steps.py connecting")
         browsersteps_sessions[browsersteps_session_id] = start_browsersteps_session(watch_uuid)
-        print("Starting connection with playwright - done")
+        logger.debug("Starting connection with playwright - done")
         return {'browsersteps_session_id': browsersteps_session_id}
 
     @login_optionally_required
@@ -189,7 +180,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                                          optional_value=step_optional_value)
 
             except Exception as e:
-                print("Exception when calling step operation", step_operation, str(e))
+                logger.error(f"Exception when calling step operation {step_operation} {str(e)}")
                 # Try to find something of value to give back to the user
                 return make_response(str(e).splitlines()[0], 401)
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -4,6 +4,7 @@ import os
 import time
 import re
 from random import randint
+from loguru import logger
 
 # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
 # 0- off, 1- on
@@ -53,7 +54,7 @@ class steppable_browser_interface():
         if call_action_name == 'choose_one':
             return
 
-        print("> action calling", call_action_name)
+        logger.debug(f"> Action calling '{call_action_name}'")
         # https://playwright.dev/python/docs/selectors#xpath-selectors
         if selector and selector.startswith('/') and not selector.startswith('//'):
             selector = "xpath=" + selector
@@ -72,7 +73,7 @@ class steppable_browser_interface():
 
         action_handler(selector, optional_value)
         self.page.wait_for_timeout(1.5 * 1000)
-        print("Call action done in", time.time() - now)
+        logger.debug(f"Call action done in {time.time()-now:.2f}s")
 
     def action_goto_url(self, selector=None, value=None):
         # self.page.set_viewport_size({"width": 1280, "height": 5000})
@@ -82,7 +83,7 @@ class steppable_browser_interface():
         #and also wait for seconds ?
         #await page.waitForTimeout(1000);
         #await page.waitForTimeout(extra_wait_ms);
-        print("Time to goto URL ", time.time() - now)
+        logger.debug(f"Time to goto URL {time.time()-now:.2f}s")
         return response
 
     def action_click_element_containing_text(self, selector=None, value=''):
@@ -103,7 +104,7 @@ class steppable_browser_interface():
         return response
 
     def action_click_element(self, selector, value):
-        print("Clicking element")
+        logger.debug("Clicking element")
         if not len(selector.strip()):
             return
 
@@ -111,7 +112,7 @@ class steppable_browser_interface():
 
     def action_click_element_if_exists(self, selector, value):
         import playwright._impl._errors as _api_types
-        print("Clicking element if exists")
+        logger.debug("Clicking element if exists")
         if not len(selector.strip()):
             return
         try:
@@ -168,7 +169,7 @@ class steppable_browser_interface():
         self.page.locator(selector, timeout=1000).uncheck(timeout=1000)
 
 
-# Responsible for maintaining a live 'context' with browserless
+# Responsible for maintaining a live 'context' with the chrome CDP
 # @todo - how long do contexts live for anyway?
 class browsersteps_live_ui(steppable_browser_interface):
     context = None
@@ -227,11 +228,11 @@ class browsersteps_live_ui(steppable_browser_interface):
         # Listen for all console events and handle errors
         self.page.on("console", lambda msg: print(f"Browser steps console - {msg.type}: {msg.text} {msg.args}"))
 
-        print("Time to browser setup", time.time() - now)
+        logger.debug(f"Time to browser setup {time.time()-now:.2f}s")
         self.page.wait_for_timeout(1 * 1000)
 
     def mark_as_closed(self):
-        print("Page closed, cleaning up..")
+        logger.debug("Page closed, cleaning up..")
 
     @property
     def has_expired(self):
@@ -257,7 +258,7 @@ class browsersteps_live_ui(steppable_browser_interface):
         xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
         # So the JS will find the smallest one first
         xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
-        print("Time to complete get_current_state of browser", time.time() - now)
+        logger.debug(f"Time to complete get_current_state of browser {time.time()-now:.2f}s")
         # except
         # playwright._impl._api_types.Error: Browser closed.
         # @todo show some countdown timer?

changedetectionio/blueprint/price_data_follower/__init__.py

@@ -18,8 +18,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q: PriorityQueue
     def accept(uuid):
         datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_ACCEPT
         update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
-        return redirect(url_for("form_watch_checknow", uuid=uuid))
-
+        return redirect(url_for("index"))
 
     @login_required
     @price_data_follower_blueprint.route("/<string:uuid>/reject", methods=['GET'])

changedetectionio/content_fetcher.py

@@ -4,12 +4,12 @@ from urllib.parse import urlparse
 import chardet
 import hashlib
 import json
-import logging
 import os
 import requests
 import sys
 import time
 import urllib.parse
+from loguru import logger
 
 visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary'
 
@@ -47,12 +47,13 @@ class BrowserStepsStepException(Exception):
     def __init__(self, step_n, original_e):
         self.step_n = step_n
         self.original_e = original_e
-        print(f"Browser Steps exception at step {self.step_n}", str(original_e))
+        logger.debug(f"Browser Steps exception at step {self.step_n} {str(original_e)}")
         return
 
 
+# @todo - make base Exception class that announces via logger()
 class PageUnloadable(Exception):
-    def __init__(self, status_code, url, message, screenshot=False):
+    def __init__(self, status_code=None, url='', message='', screenshot=False):
         # Set this so we can use it in other parts of the app
         self.status_code = status_code
         self.url = url
@@ -60,6 +61,10 @@ class PageUnloadable(Exception):
         self.message = message
         return
 
+class BrowserStepsInUnsupportedFetcher(Exception):
+    def __init__(self, url):
+        self.url = url
+        return
 
 class EmptyReply(Exception):
     def __init__(self, status_code, url, screenshot=None):
@@ -188,7 +193,7 @@ class Fetcher():
 
             for step in valid_steps:
                 step_n += 1
-                print(">> Iterating check - browser Step n {} - {}...".format(step_n, step['operation']))
+                logger.debug(f">> Iterating check - browser Step n {step_n} - {step['operation']}...")
                 self.screenshot_step("before-" + str(step_n))
                 self.save_step_html("before-" + str(step_n))
                 try:
@@ -205,8 +210,8 @@ class Fetcher():
                                                       optional_value=optional_value)
                     self.screenshot_step(step_n)
                     self.save_step_html(step_n)
-
                 except (Error, TimeoutError) as e:
+                    logger.debug(str(e))
                     # Stop processing here
                     raise BrowserStepsStepException(step_n=step_n, original_e=e)
 
@@ -295,128 +300,17 @@ class base_html_playwright(Fetcher):
 
         if self.browser_steps_screenshot_path is not None:
             destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.jpeg'.format(step_n))
-            logging.debug("Saving step screenshot to {}".format(destination))
+            logger.debug(f"Saving step screenshot to {destination}")
             with open(destination, 'wb') as f:
                 f.write(screenshot)
 
     def save_step_html(self, step_n):
         content = self.page.content()
         destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
-        logging.debug("Saving step HTML to {}".format(destination))
+        logger.debug(f"Saving step HTML to {destination}")
         with open(destination, 'w') as f:
             f.write(content)
 
-    def run_fetch_browserless_puppeteer(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
-
-        from pkg_resources import resource_string
-
-        extra_wait_ms = (int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay) * 1000
-
-        self.xpath_element_js = self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
-        code = resource_string(__name__, "res/puppeteer_fetch.js").decode('utf-8')
-        # In the future inject this is a proper JS package
-        code = code.replace('%xpath_scrape_code%', self.xpath_element_js)
-        code = code.replace('%instock_scrape_code%', self.instock_data_js)
-
-        from requests.exceptions import ConnectTimeout, ReadTimeout
-        wait_browserless_seconds = 240
-
-        browserless_function_url = os.getenv('BROWSERLESS_FUNCTION_URL')
-        from urllib.parse import urlparse
-        if not browserless_function_url:
-            # Convert/try to guess from PLAYWRIGHT_DRIVER_URL
-            o = urlparse(os.getenv('PLAYWRIGHT_DRIVER_URL'))
-            browserless_function_url = o._replace(scheme="http")._replace(path="function").geturl()
-
-
-        # Append proxy connect string
-        if self.proxy:
-            # Remove username/password if it exists in the URL or you will receive "ERR_NO_SUPPORTED_PROXIES" error
-            # Actual authentication handled by Puppeteer/node
-            o = urlparse(self.proxy.get('server'))
-            proxy_url = urllib.parse.quote(o._replace(netloc="{}:{}".format(o.hostname, o.port)).geturl())
-            browserless_function_url = f"{browserless_function_url}&--proxy-server={proxy_url}"
-
-        try:
-            amp = '&' if '?' in browserless_function_url else '?'
-            response = requests.request(
-                method="POST",
-                json={
-                    "code": code,
-                    "context": {
-                        # Very primitive disk cache - USE WITH EXTREME CAUTION
-                        # Run browserless container  with -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]"
-                        'disk_cache_dir': os.getenv("PUPPETEER_DISK_CACHE", False), # or path to disk cache ending in /, ie /tmp/cache/
-                        'execute_js': self.webdriver_js_execute_code,
-                        'extra_wait_ms': extra_wait_ms,
-                        'include_filters': current_include_filters,
-                        'req_headers': request_headers,
-                        'screenshot_quality': int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)),
-                        'url': url,
-                        'user_agent': {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None),
-                        'proxy_username': self.proxy.get('username', '') if self.proxy else False,
-                        'proxy_password': self.proxy.get('password', '') if self.proxy and self.proxy.get('username') else False,
-                        'no_cache_list': [
-                            'twitter',
-                            '.pdf'
-                        ],
-                        # Could use https://github.com/easylist/easylist here, or install a plugin
-                        'block_url_list': [
-                            'adnxs.com',
-                            'analytics.twitter.com',
-                            'doubleclick.net',
-                            'google-analytics.com',
-                            'googletagmanager',
-                            'trustpilot.com'
-                        ]
-                    }
-                },
-                # @todo /function needs adding ws:// to http:// rebuild this
-                url=browserless_function_url+f"{amp}--disable-features=AudioServiceOutOfProcess&dumpio=true&--disable-remote-fonts",
-                timeout=wait_browserless_seconds)
-
-        except ReadTimeout:
-            raise PageUnloadable(url=url, status_code=None, message=f"No response from browserless in {wait_browserless_seconds}s")
-        except ConnectTimeout:
-            raise PageUnloadable(url=url, status_code=None, message=f"Timed out connecting to browserless, retrying..")
-        else:
-            # 200 Here means that the communication to browserless worked only, not the page state
-            if response.status_code == 200:
-                import base64
-
-                x = response.json()
-                if not x.get('screenshot'):
-                    # https://github.com/puppeteer/puppeteer/blob/v1.0.0/docs/troubleshooting.md#tips
-                    # https://github.com/puppeteer/puppeteer/issues/1834
-                    # https://github.com/puppeteer/puppeteer/issues/1834#issuecomment-381047051
-                    # Check your memory is shared and big enough
-                    raise ScreenshotUnavailable(url=url, status_code=None)
-
-                if not x.get('content', '').strip():
-                    raise EmptyReply(url=url, status_code=None)
-
-                if x.get('status_code', 200) != 200 and not ignore_status_codes:
-                    raise Non200ErrorCodeReceived(url=url, status_code=x.get('status_code', 200), page_html=x['content'])
-
-                self.content = x.get('content')
-                self.headers = x.get('headers')
-                self.instock_data = x.get('instock_data')
-                self.screenshot = base64.b64decode(x.get('screenshot'))
-                self.status_code = x.get('status_code')
-                self.xpath_data = x.get('xpath_data')
-
-            else:
-                # Some other error from browserless
-                raise PageUnloadable(url=url, status_code=None, message=response.content.decode('utf-8'))
-
     def run(self,
             url,
             timeout,
@@ -428,21 +322,6 @@ class base_html_playwright(Fetcher):
             is_binary=False):
 
 
-        # For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!)
-        # browser_connection_is_custom doesnt work with puppeteer style fetch (use playwright native too in this case)
-        if not self.browser_connection_is_custom and not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
-            if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')):
-                # Temporary backup solution until we rewrite the playwright code
-                return self.run_fetch_browserless_puppeteer(
-                    url,
-                    timeout,
-                    request_headers,
-                    request_body,
-                    request_method,
-                    ignore_status_codes,
-                    current_include_filters,
-                    is_binary)
-
         from playwright.sync_api import sync_playwright
         import playwright._impl._errors
 
@@ -491,7 +370,7 @@ class base_html_playwright(Fetcher):
             if response is None:
                 context.close()
                 browser.close()
-                print("Content Fetcher > Response object was none")
+                logger.debug("Content Fetcher > Response object was none")
                 raise EmptyReply(url=url, status_code=None)
 
             try:
@@ -503,7 +382,7 @@ class base_html_playwright(Fetcher):
                 # This can be ok, we will try to grab what we could retrieve
                 pass
             except Exception as e:
-                print("Content Fetcher > Other exception when executing custom JS code", str(e))
+                logger.debug(f"Content Fetcher > Other exception when executing custom JS code {str(e)}")
                 context.close()
                 browser.close()
                 raise PageUnloadable(url=url, status_code=None, message=str(e))
@@ -511,8 +390,15 @@ class base_html_playwright(Fetcher):
             extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
             self.page.wait_for_timeout(extra_wait * 1000)
 
-
-            self.status_code = response.status
+            try:
+                self.status_code = response.status
+            except Exception as e:
+                # https://github.com/dgtlmoon/changedetection.io/discussions/2122#discussioncomment-8241962
+                logger.critical(f"Response from the browser/Playwright did not have a status_code! Response follows.")
+                logger.critical(response)
+                context.close()
+                browser.close()
+                raise PageUnloadable(url=url, status_code=None, message=str(e))
 
             if self.status_code != 200 and not ignore_status_codes:
 
@@ -524,7 +410,7 @@ class base_html_playwright(Fetcher):
             if len(self.page.content().strip()) == 0:
                 context.close()
                 browser.close()
-                print("Content Fetcher > Content was empty")
+                logger.debug("Content Fetcher > Content was empty")
                 raise EmptyReply(url=url, status_code=response.status)
 
             # Run Browser Steps here
@@ -552,16 +438,17 @@ class base_html_playwright(Fetcher):
             # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
             # acceptable screenshot quality here
             try:
-                # The actual screenshot
-                self.screenshot = self.page.screenshot(type='jpeg', full_page=True,
-                                                       quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
+                # The actual screenshot - this always base64 and needs decoding! horrible! huge CPU usage
+                self.screenshot = self.page.screenshot(type='jpeg',
+                                                       full_page=True,
+                                                       quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)),
+                                                       )
             except Exception as e:
+                # It's likely the screenshot was too long/big and something crashed
+                raise ScreenshotUnavailable(url=url, status_code=self.status_code)
+            finally:
                 context.close()
                 browser.close()
-                raise ScreenshotUnavailable(url=url, status_code=response.status_code)
-
-            context.close()
-            browser.close()
 
 
 class base_html_webdriver(Fetcher):
@@ -676,7 +563,7 @@ class base_html_webdriver(Fetcher):
             try:
                 self.driver.quit()
             except Exception as e:
-                print("Content Fetcher > Exception in chrome shutdown/quit" + str(e))
+                logger.debug(f"Content Fetcher > Exception in chrome shutdown/quit {str(e)}")
 
 
 # "html_requests" is listed as the default fetcher in store.py!
@@ -698,6 +585,9 @@ class html_requests(Fetcher):
             current_include_filters=None,
             is_binary=False):
 
+        if self.browser_steps_get_valid_steps():
+            raise BrowserStepsInUnsupportedFetcher(url=url)
+
         # Make requests use a more modern looking user-agent
         if not {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None):
             request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT",
@@ -737,6 +627,8 @@ class html_requests(Fetcher):
                 if encoding:
                     r.encoding = encoding
 
+        self.headers = r.headers
+
         if not r.content or not len(r.content):
             raise EmptyReply(url=url, status_code=r.status_code)
 
@@ -753,7 +645,7 @@ class html_requests(Fetcher):
         else:
             self.content = r.text
 
-        self.headers = r.headers
+
         self.raw_content = r.content
 
 

changedetectionio/flask_app.py

@@ -12,7 +12,7 @@ from functools import wraps
 from threading import Event
 import datetime
 import flask_login
-import logging
+from loguru import logger
 import os
 import pytz
 import queue
@@ -210,6 +210,8 @@ def login_optionally_required(func):
     return decorated_view
 
 def changedetection_app(config=None, datastore_o=None):
+    logger.trace("TRACE log is enabled")
+
     global datastore
     datastore = datastore_o
 
@@ -314,6 +316,9 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/rss", methods=['GET'])
     def rss():
+        from jinja2 import Environment, BaseLoader
+        jinja2_env = Environment(loader=BaseLoader)
+        now = time.time()
         # Always requires token set
         app_rss_token = datastore.data['settings']['application'].get('rss_access_token')
         rss_url_token = request.args.get('token')
@@ -377,8 +382,12 @@ def changedetection_app(config=None, datastore_o=None):
                                              include_equal=False,
                                              line_feed_sep="<br>")
 
-                fe.content(content="<html><body><h4>{}</h4>{}</body></html>".format(watch_title, html_diff),
-                           type='CDATA')
+                # @todo Make this configurable and also consider html-colored markup
+                # @todo User could decide if <link> goes to the diff page, or to the watch link
+                rss_template = "<html><body>\n<h4><a href=\"{{watch_url}}\">{{watch_title}}</a></h4>\n<p>{{html_diff}}</p>\n</body></html>\n"
+                content = jinja2_env.from_string(rss_template).render(watch_title=watch_title, html_diff=html_diff, watch_url=watch.link)
+
+                fe.content(content=content, type='CDATA')
 
                 fe.guid(guid, permalink=False)
                 dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key))
@@ -387,6 +396,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         response = make_response(fg.rss_str())
         response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8')
+        logger.trace(f"RSS generated in {time.time() - now:.3f}s")
         return response
 
     @app.route("/", methods=['GET'])
@@ -1492,7 +1502,7 @@ def changedetection_app(config=None, datastore_o=None):
 
 
         except Exception as e:
-            logging.error("Error sharing -{}".format(str(e)))
+            logger.error(f"Error sharing -{str(e)}")
             flash("Could not share, something went wrong while communicating with the share server - {}".format(str(e)), 'error')
 
         # https://changedetection.io/share/VrMv05wpXyQa
@@ -1600,12 +1610,12 @@ def notification_runner():
                     n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
 
                 if not n_object.get('notification_format') and datastore.data['settings']['application'].get('notification_format'):
-                    n_object['notification_title'] = datastore.data['settings']['application'].get('notification_format')
+                    n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
 
                 sent_obj = notification.process_notification(n_object, datastore)
 
             except Exception as e:
-                logging.error("Watch URL: {}  Error {}".format(n_object['watch_url'], str(e)))
+                logger.error(f"Watch URL: {n_object['watch_url']}  Error {str(e)}")
 
                 # UUID wont be present when we submit a 'test' from the global settings
                 if 'uuid' in n_object:
@@ -1628,7 +1638,7 @@ def ticker_thread_check_time_launch_checks():
     proxy_last_called_time = {}
 
     recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 20))
-    print("System env MINIMUM_SECONDS_RECHECK_TIME", recheck_time_minimum_seconds)
+    logger.debug(f"System env MINIMUM_SECONDS_RECHECK_TIME {recheck_time_minimum_seconds}")
 
     # Spin up Workers that do the fetching
     # Can be overriden by ENV or use the default settings
@@ -1673,7 +1683,7 @@ def ticker_thread_check_time_launch_checks():
             now = time.time()
             watch = datastore.data['watching'].get(uuid)
             if not watch:
-                logging.error("Watch: {} no longer present.".format(uuid))
+                logger.error(f"Watch: {uuid} no longer present.")
                 continue
 
             # No need todo further processing if it's paused
@@ -1706,10 +1716,10 @@ def ticker_thread_check_time_launch_checks():
                             time_since_proxy_used = int(time.time() - proxy_last_used_time)
                             if time_since_proxy_used < proxy_list_reuse_time_minimum:
                                 # Not enough time difference reached, skip this watch
-                                print("> Skipped UUID {} using proxy '{}', not enough time between proxy requests {}s/{}s".format(uuid,
-                                                                                                                         watch_proxy,
-                                                                                                                         time_since_proxy_used,
-                                                                                                                         proxy_list_reuse_time_minimum))
+                                logger.debug(f"> Skipped UUID {uuid} "
+                                        f"using proxy '{watch_proxy}', not "
+                                        f"enough time between proxy requests "
+                                        f"{time_since_proxy_used}s/{proxy_list_reuse_time_minimum}s")
                                 continue
                             else:
                                 # Record the last used time
@@ -1717,14 +1727,12 @@ def ticker_thread_check_time_launch_checks():
 
                     # Use Epoch time as priority, so we get a "sorted" PriorityQueue, but we can still push a priority 1 into it.
                     priority = int(time.time())
-                    print(
-                        "> Queued watch UUID {} last checked at {} queued at {:0.2f} priority {} jitter {:0.2f}s, {:0.2f}s since last checked".format(
-                            uuid,
-                            watch['last_checked'],
-                            now,
-                            priority,
-                            watch.jitter_seconds,
-                            now - watch['last_checked']))
+                    logger.debug(
+                        f"> Queued watch UUID {uuid} "
+                        f"last checked at {watch['last_checked']} "
+                        f"queued at {now:0.2f} priority {priority} "
+                        f"jitter {watch.jitter_seconds:0.2f}s, "
+                        f"{now - watch['last_checked']:0.2f}s since last checked")
 
                     # Into the queue with you
                     update_q.put(queuedWatchMetaData.PrioritizedItem(priority=priority, item={'uuid': uuid, 'skip_when_checksum_same': True}))

changedetectionio/forms.py

@@ -465,6 +465,7 @@ class watchForm(commonSettingsForm):
     method = SelectField('Request method', choices=valid_method, default=default_method)
     ignore_status_codes = BooleanField('Ignore status codes (process non-2xx status codes as normal)', default=False)
     check_unique_lines = BooleanField('Only trigger when unique lines appear', default=False)
+    sort_text_alphabetically =  BooleanField('Sort text alphabetically', default=False)
 
     filter_text_added = BooleanField('Added lines', default=True)
     filter_text_replaced = BooleanField('Replaced/changed lines', default=True)

changedetectionio/importer.py

@@ -2,6 +2,7 @@ from abc import ABC, abstractmethod
 import time
 import validators
 from wtforms import ValidationError
+from loguru import logger
 
 from changedetectionio.forms import validate_url
 
@@ -56,7 +57,7 @@ class import_url_list(Importer):
 
             # Flask wtform validators wont work with basic auth, use validators package
             # Up to 5000 per batch so we dont flood the server
-            # @todo validators.url failed on local hostnames (such as referring to ourself when using browserless)
+            # @todo validators.url will fail when you add your own IP etc
             if len(url) and 'http' in url.lower() and good < 5000:
                 extras = None
                 if processor:
@@ -195,7 +196,7 @@ class import_xlsx_wachete(Importer):
                     try:
                         validate_url(data.get('url'))
                     except ValidationError as e:
-                        print(">> import URL error", data.get('url'), str(e))
+                        logger.error(f">> Import URL error {data.get('url')} {str(e)}")
                         flash(f"Error processing row number {row_id}, URL value was incorrect, row was skipped.", 'error')
                         # Don't bother processing anything else on this row
                         continue
@@ -209,7 +210,7 @@ class import_xlsx_wachete(Importer):
                         self.new_uuids.append(new_uuid)
                         good += 1
             except Exception as e:
-                print(e)
+                logger.error(e)
                 flash(f"Error processing row number {row_id}, check all cell data types are correct, row was skipped.", 'error')
             else:
                 row_id += 1
@@ -264,7 +265,7 @@ class import_xlsx_custom(Importer):
                         try:
                             validate_url(url)
                         except ValidationError as e:
-                            print(">> Import URL error", url, str(e))
+                            logger.error(f">> Import URL error {url} {str(e)}")
                             flash(f"Error processing row number {row_i}, URL value was incorrect, row was skipped.", 'error')
                             # Don't bother processing anything else on this row
                             url = None
@@ -293,7 +294,7 @@ class import_xlsx_custom(Importer):
                         self.new_uuids.append(new_uuid)
                         good += 1
         except Exception as e:
-            print(e)
+            logger.error(e)
             flash(f"Error processing row number {row_i}, check all cell data types are correct, row was skipped.", 'error')
         else:
             row_i += 1

changedetectionio/model/Watch.py

@@ -1,10 +1,10 @@
 from distutils.util import strtobool
-import logging
 import os
 import re
 import time
 import uuid
 from pathlib import Path
+from loguru import logger
 
 # Allowable protocols, protects against javascript: etc
 # file:// is further checked by ALLOW_FILE_URI
@@ -45,6 +45,7 @@ base_config = {
     'last_error': False,
     'last_viewed': 0,  # history key value of the last viewed via the [diff] link
     'method': 'GET',
+    'notification_alert_count': 0,
     # Custom notification content
     'notification_body': None,
     'notification_format': default_notification_format_for_watch,
@@ -56,6 +57,8 @@ base_config = {
     'previous_md5': False,
     'previous_md5_before_filters': False,  # Used for skipping changedetection entirely
     'proxy': None,  # Preferred proxy connection
+    'remote_server_reply': None, # From 'server' reply header
+    'sort_text_alphabetically': False,
     'subtractive_selectors': [],
     'tag': '', # Old system of text name for a tag, to be removed
     'tags': [], # list of UUIDs to App.Tags
@@ -122,7 +125,7 @@ class model(dict):
 
     def ensure_data_dir_exists(self):
         if not os.path.isdir(self.watch_data_dir):
-            print ("> Creating data dir {}".format(self.watch_data_dir))
+            logger.debug(f"> Creating data dir {self.watch_data_dir}")
             os.mkdir(self.watch_data_dir)
 
     @property
@@ -211,7 +214,7 @@ class model(dict):
         # Read the history file as a dict
         fname = os.path.join(self.watch_data_dir, "history.txt")
         if os.path.isfile(fname):
-            logging.debug("Reading history index " + str(time.time()))
+            logger.debug(f"Reading watch history index for {self.get('uuid')}")
             with open(fname, "r") as f:
                 for i in f.readlines():
                     if ',' in i:
@@ -246,10 +249,10 @@ class model(dict):
     @property
     def has_browser_steps(self):
         has_browser_steps = self.get('browser_steps') and list(filter(
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
-                self.get('browser_steps')))
+            lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
+            self.get('browser_steps')))
 
-        return  has_browser_steps
+        return has_browser_steps
 
     # Returns the newest key, but if theres only 1 record, then it's counted as not being new, so return 0.
     @property

changedetectionio/notification.py

@@ -1,7 +1,9 @@
 import apprise
+import time
 from jinja2 import Environment, BaseLoader
 from apprise import NotifyFormat
 import json
+from loguru import logger
 
 valid_tokens = {
     'base_url': '',
@@ -114,6 +116,9 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
 
 def process_notification(n_object, datastore):
 
+    now = time.time()
+    if n_object.get('notification_timestamp'):
+        logger.trace(f"Time since queued {now-n_object['notification_timestamp']:.3f}s")
     # Insert variables into the notification content
     notification_parameters = create_notification_parameters(n_object, datastore)
 
@@ -131,90 +136,100 @@ def process_notification(n_object, datastore):
         # Initially text or whatever
         n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
 
+    logger.trace(f"Complete notification body including Jinja and placeholders calculated in  {time.time() - now:.3f}s")
 
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
     # Anything higher than or equal to WARNING (which covers things like Connection errors)
     # raise it as an exception
-    apobjs=[]
-    sent_objs=[]
+
+    sent_objs = []
     from .apprise_asset import asset
-    for url in n_object['notification_urls']:
-        url = jinja2_env.from_string(url).render(**notification_parameters)
-        apobj = apprise.Apprise(debug=True, asset=asset)
-        url = url.strip()
-        if len(url):
-            print(">> Process Notification: AppRise notifying {}".format(url))
-            with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
-                # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
-                # Because different notifications may require different pre-processing, run each sequentially :(
-                # 2000 bytes minus -
-                #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
-                #     Length of URL - Incase they specify a longer custom avatar_url
+    apobj = apprise.Apprise(debug=True, asset=asset)
 
-                # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
-                k = '?' if not '?' in url else '&'
-                if not 'avatar_url' in url \
-                        and not url.startswith('mail') \
-                        and not url.startswith('post') \
-                        and not url.startswith('get') \
-                        and not url.startswith('delete') \
-                        and not url.startswith('put'):
-                    url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
+    if not n_object.get('notification_urls'):
+        return None
 
-                if url.startswith('tgram://'):
-                    # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
-                    # re https://github.com/dgtlmoon/changedetection.io/issues/555
-                    # @todo re-use an existing library we have already imported to strip all non-allowed tags
-                    n_body = n_body.replace('<br>', '\n')
-                    n_body = n_body.replace('</br>', '\n')
-                    # real limit is 4096, but minus some for extra metadata
-                    payload_max_size = 3600
-                    body_limit = max(0, payload_max_size - len(n_title))
-                    n_title = n_title[0:payload_max_size]
-                    n_body = n_body[0:body_limit]
+    with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
+        for url in n_object['notification_urls']:
+            url = url.strip()
+            if not url:
+                logger.warning(f"Process Notification: skipping empty notification URL.")
+                continue
 
-                elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith('https://discord.com/api'):
-                    # real limit is 2000, but minus some for extra metadata
-                    payload_max_size = 1700
-                    body_limit = max(0, payload_max_size - len(n_title))
-                    n_title = n_title[0:payload_max_size]
-                    n_body = n_body[0:body_limit]
+            logger.info(">> Process Notification: AppRise notifying {}".format(url))
+            url = jinja2_env.from_string(url).render(**notification_parameters)
 
-                elif url.startswith('mailto'):
-                    # Apprise will default to HTML, so we need to override it
-                    # So that whats' generated in n_body is in line with what is going to be sent.
-                    # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
-                    if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
-                        prefix = '?' if not '?' in url else '&'
-                        # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
-                        n_format = n_format.lower()
-                        url = f"{url}{prefix}format={n_format}"
-                    # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
+            # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
+            # Because different notifications may require different pre-processing, run each sequentially :(
+            # 2000 bytes minus -
+            #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
+            #     Length of URL - Incase they specify a longer custom avatar_url
 
-                apobj.add(url)
+            # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
+            k = '?' if not '?' in url else '&'
+            if not 'avatar_url' in url \
+                    and not url.startswith('mail') \
+                    and not url.startswith('post') \
+                    and not url.startswith('get') \
+                    and not url.startswith('delete') \
+                    and not url.startswith('put'):
+                url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
 
-                apobj.notify(
-                    title=n_title,
-                    body=n_body,
-                    body_format=n_format,
-                    # False is not an option for AppRise, must be type None
-                    attach=n_object.get('screenshot', None)
-                )
+            if url.startswith('tgram://'):
+                # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
+                # re https://github.com/dgtlmoon/changedetection.io/issues/555
+                # @todo re-use an existing library we have already imported to strip all non-allowed tags
+                n_body = n_body.replace('<br>', '\n')
+                n_body = n_body.replace('</br>', '\n')
+                # real limit is 4096, but minus some for extra metadata
+                payload_max_size = 3600
+                body_limit = max(0, payload_max_size - len(n_title))
+                n_title = n_title[0:payload_max_size]
+                n_body = n_body[0:body_limit]
 
-                apobj.clear()
+            elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith(
+                    'https://discord.com/api'):
+                # real limit is 2000, but minus some for extra metadata
+                payload_max_size = 1700
+                body_limit = max(0, payload_max_size - len(n_title))
+                n_title = n_title[0:payload_max_size]
+                n_body = n_body[0:body_limit]
 
-                # Incase it needs to exist in memory for a while after to process(?)
-                apobjs.append(apobj)
+            elif url.startswith('mailto'):
+                # Apprise will default to HTML, so we need to override it
+                # So that whats' generated in n_body is in line with what is going to be sent.
+                # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
+                if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
+                    prefix = '?' if not '?' in url else '&'
+                    # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
+                    n_format = n_format.lower()
+                    url = f"{url}{prefix}format={n_format}"
+                # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
 
-                # Returns empty string if nothing found, multi-line string otherwise
-                log_value = logs.getvalue()
-                if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
-                    raise Exception(log_value)
+            apobj.add(url)
 
-                sent_objs.append({'title': n_title,
-                                  'body': n_body,
-                                  'url' : url,
-                                  'body_format': n_format})
+            sent_objs.append({'title': n_title,
+                              'body': n_body,
+                              'url': url,
+                              'body_format': n_format})
+
+        # Blast off the notifications tht are set in .add()
+        apobj.notify(
+            title=n_title,
+            body=n_body,
+            body_format=n_format,
+            # False is not an option for AppRise, must be type None
+            attach=n_object.get('screenshot', None)
+        )
+
+        # Give apprise time to register an error
+        time.sleep(3)
+
+        # Returns empty string if nothing found, multi-line string otherwise
+        log_value = logs.getvalue()
+
+        if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
+            raise Exception(log_value)
 
     # Return what was sent for better logging - after the for loop
     return sent_objs

changedetectionio/processors/__init__.py

@@ -5,6 +5,7 @@ import re
 from changedetectionio import content_fetcher
 from copy import deepcopy
 from distutils.util import strtobool
+from loguru import logger
 
 class difference_detection_processor():
 
@@ -69,7 +70,7 @@ class difference_detection_processor():
         proxy_url = None
         if preferred_proxy_id:
             proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')
-            print(f"Using proxy Key: {preferred_proxy_id} as Proxy URL {proxy_url}")
+            logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}' for {url}")
 
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)

changedetectionio/processors/restock_diff.py

@@ -1,8 +1,9 @@
 
-import hashlib
-import urllib3
 from . import difference_detection_processor
 from copy import deepcopy
+from loguru import logger
+import hashlib
+import urllib3
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -43,11 +44,13 @@ class perform_site_check(difference_detection_processor):
             fetched_md5 = hashlib.md5(self.fetcher.instock_data.encode('utf-8')).hexdigest()
             # 'Possibly in stock' comes from stock-not-in-stock.js when no string found above the fold.
             update_obj["in_stock"] = True if self.fetcher.instock_data == 'Possibly in stock' else False
+            logger.debug(f"Watch UUID {uuid} restock check returned '{self.fetcher.instock_data}' from JS scraper.")
         else:
             raise UnableToExtractRestockData(status_code=self.fetcher.status_code)
 
         # The main thing that all this at the moment comes down to :)
         changed_detected = False
+        logger.debug(f"Watch UUID {uuid} restock check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
 
         if watch.get('previous_md5') and watch.get('previous_md5') != fetched_md5:
             # Yes if we only care about it going to instock, AND we are in stock
@@ -60,5 +63,4 @@ class perform_site_check(difference_detection_processor):
 
         # Always record the new checksum
         update_obj["previous_md5"] = fetched_md5
-
-        return changed_detected, update_obj, self.fetcher.instock_data.encode('utf-8')
+        return changed_detected, update_obj, self.fetcher.instock_data.encode('utf-8').strip()

changedetectionio/processors/text_json_diff.py

@@ -2,16 +2,16 @@
 
 import hashlib
 import json
-import logging
 import os
 import re
 import urllib3
 
+from . import difference_detection_processor
+from ..html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text
 from changedetectionio import content_fetcher, html_tools
 from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
 from copy import deepcopy
-from . import difference_detection_processor
-from ..html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text
+from loguru import logger
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -116,7 +116,9 @@ class perform_site_check(difference_detection_processor):
         # and then use getattr https://docs.python.org/3/reference/datamodel.html#object.__getitem__
         # https://realpython.com/inherit-python-dict/ instead of doing it procedurely
         include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='include_filters')
-        include_filters_rule = [*watch.get('include_filters', []), *include_filters_from_tags]
+
+        # 1845 - remove duplicated filters in both group and watch include filter
+        include_filters_rule = list({*watch.get('include_filters', []), *include_filters_from_tags})
 
         subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='subtractive_selectors'),
                                  *watch.get("subtractive_selectors", []),
@@ -202,6 +204,12 @@ class perform_site_check(difference_detection_processor):
                             is_rss=is_rss # #1874 activate the <title workaround hack
                         )
 
+        if watch.get('sort_text_alphabetically') and stripped_text_from_html:
+            # Note: Because a <p>something</p> will add an extra line feed to signify the paragraph gap
+            # we end up with 'Some text\n\n', sorting will add all those extra \n at the start, so we remove them here.
+            stripped_text_from_html = stripped_text_from_html.replace('\n\n', '\n')
+            stripped_text_from_html = '\n'.join( sorted(stripped_text_from_html.splitlines(), key=lambda x: x.lower() ))
+
         # Re #340 - return the content before the 'ignore text' was applied
         text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 
@@ -335,15 +343,17 @@ class perform_site_check(difference_detection_processor):
                 if not watch['title'] or not len(watch['title']):
                     update_obj['title'] = html_tools.extract_element(find='title', html_content=self.fetcher.content)
 
+        logger.debug(f"Watch UUID {uuid} content check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
+
         if changed_detected:
             if watch.get('check_unique_lines', False):
                 has_unique_lines = watch.lines_contain_something_unique_compared_to_history(lines=stripped_text_from_html.splitlines())
                 # One or more lines? unsure?
                 if not has_unique_lines:
-                    logging.debug("check_unique_lines: UUID {} didnt have anything new setting change_detected=False".format(uuid))
+                    logger.debug(f"check_unique_lines: UUID {uuid} didnt have anything new setting change_detected=False")
                     changed_detected = False
                 else:
-                    logging.debug("check_unique_lines: UUID {} had unique content".format(uuid))
+                    logger.debug(f"check_unique_lines: UUID {uuid} had unique content")
 
         # Always record the new checksum
         update_obj["previous_md5"] = fetched_md5

changedetectionio/res/puppeteer_fetch.js

@@ -146,7 +146,7 @@ module.exports = async ({page, context}) => {
     var xpath_data;
     var instock_data;
     try {
-        // Not sure the best way here, in the future this should be a new package added to npm then run in browserless
+        // Not sure the best way here, in the future this should be a new package added to npm then run in evaluatedCode
         // (Once the old playwright is removed)
         xpath_data = await page.evaluate((include_filters) => {%xpath_scrape_code%}, include_filters);
         instock_data = await page.evaluate(() => {%instock_scrape_code%});

changedetectionio/res/stock-not-in-stock.js

@@ -1,117 +1,134 @@
+// Restock Detector
+// (c) Leigh Morresi dgtlmoon@gmail.com
+//
+// Assumes the product is in stock to begin with, unless the following appears above the fold ;
+// - outOfStockTexts appears above the fold (out of stock)
+// - negateOutOfStockRegex (really is in stock)
+
 function isItemInStock() {
-  // @todo Pass these in so the same list can be used in non-JS fetchers
-  const outOfStockTexts = [
-    ' أخبرني عندما يتوفر',
-    '0 in stock',
-    'agotado',
-    'article épuisé',
-    'artikel zurzeit vergriffen',
-    'as soon as stock is available',
-    'ausverkauft', // sold out
-    'available for back order',
-    'back-order or out of stock',
-    'backordered',
-    'benachrichtigt mich', // notify me
-    'brak na stanie',
-    'brak w magazynie',
-    'coming soon',
-    'currently have any tickets for this',
-    'currently unavailable',
-    'dostępne wkrótce',
-    'en rupture de stock',
-    'ist derzeit nicht auf lager',
-    'item is no longer available',
-    'let me know when it\'s available',
-    'message if back in stock',
-    'nachricht bei',
-    'nicht auf lager',
-    'nicht lieferbar',
-    'nicht zur verfügung',
-    'niet beschikbaar',
-    'niet leverbaar',
-    'no disponible temporalmente',
-    'no longer in stock',
-    'no tickets available',
-    'not available',
-    'not currently available',
-    'not in stock',
-    'notify me when available',
-    'não estamos a aceitar encomendas',
-    'out of stock',
-    'out-of-stock',
-    'produkt niedostępny',
-    'sold out',
-    'sold-out',
-    'temporarily out of stock',
-    'temporarily unavailable',
-    'tickets unavailable',
-    'tijdelijk uitverkocht',
-    'unavailable tickets',
-    'we do not currently have an estimate of when this product will be back in stock.',
-    'zur zeit nicht an lager',
-    '品切れ',
-    '已售完',
-    '품절'
-  ];
+    // @todo Pass these in so the same list can be used in non-JS fetchers
+    const outOfStockTexts = [
+        ' أخبرني عندما يتوفر',
+        '0 in stock',
+        'agotado',
+        'article épuisé',
+        'artikel zurzeit vergriffen',
+        'as soon as stock is available',
+        'ausverkauft', // sold out
+        'available for back order',
+        'back-order or out of stock',
+        'backordered',
+        'benachrichtigt mich', // notify me
+        'brak na stanie',
+        'brak w magazynie',
+        'coming soon',
+        'currently have any tickets for this',
+        'currently unavailable',
+        'dostępne wkrótce',
+        'en rupture de stock',
+        'ist derzeit nicht auf lager',
+        'item is no longer available',
+        'let me know when it\'s available',
+        'message if back in stock',
+        'nachricht bei',
+        'nicht auf lager',
+        'nicht lieferbar',
+        'nicht zur verfügung',
+        'niet beschikbaar',
+        'niet leverbaar',
+        'niet op voorraad',
+        'no disponible temporalmente',
+        'no longer in stock',
+        'no tickets available',
+        'not available',
+        'not currently available',
+        'not in stock',        
+        'notify me when available',
+        'notify when available',            
+        'não estamos a aceitar encomendas',
+        'out of stock',
+        'out-of-stock',
+        'prodotto esaurito',
+        'produkt niedostępny',
+        'sold out',
+        'sold-out',
+        'temporarily out of stock',
+        'temporarily unavailable',
+        'tickets unavailable',
+        'tijdelijk uitverkocht',
+        'unavailable tickets',
+        'we do not currently have an estimate of when this product will be back in stock.',
+        'we don\'t know when or if this item will be back in stock.',
+        'zur zeit nicht an lager',
+        '品切れ',
+        '已售完',
+        '품절'
+    ];
 
-
-  const negateOutOfStockRegexs = [
-      '[0-9] in stock'
-  ]
-  var negateOutOfStockRegexs_r = [];
-  for (let i = 0; i < negateOutOfStockRegexs.length; i++) {
-    negateOutOfStockRegexs_r.push(new RegExp(negateOutOfStockRegexs[0], 'g'));
-  }
-
-
-  const elementsWithZeroChildren = Array.from(document.getElementsByTagName('*')).filter(element => element.children.length === 0);
-
-  // REGEXS THAT REALLY MEAN IT'S IN STOCK
-  for (let i = elementsWithZeroChildren.length - 1; i >= 0; i--) {
-    const element = elementsWithZeroChildren[i];
-    if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) {
-      var elementText="";
-      if (element.tagName.toLowerCase() === "input") {
-        elementText = element.value.toLowerCase();
-      } else {
-        elementText = element.textContent.toLowerCase();
-      }
-
-      if (elementText.length) {
-        // try which ones could mean its in stock
-        for (let i = 0; i < negateOutOfStockRegexs.length; i++) {
-          if (negateOutOfStockRegexs_r[i].test(elementText)) {
-            return 'Possibly in stock';
-          }
-        }
-      }
+    const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+    function getElementBaseText(element) {
+        // .textContent can include text from children which may give the wrong results
+        // scan only immediate TEXT_NODEs, which will be a child of the element
+        var text = "";
+        for (var i = 0; i < element.childNodes.length; ++i)
+            if (element.childNodes[i].nodeType === Node.TEXT_NODE)
+                text += element.childNodes[i].textContent;
+        return text.toLowerCase().trim();
     }
-  }
 
-  // OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK
-  for (let i = elementsWithZeroChildren.length - 1; i >= 0; i--) {
-    const element = elementsWithZeroChildren[i];
-    if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) {
-      var elementText="";
-      if (element.tagName.toLowerCase() === "input") {
-        elementText = element.value.toLowerCase();
-      } else {
-        elementText = element.textContent.toLowerCase();
-      }
+    const negateOutOfStockRegex = new RegExp('([0-9] in stock|add to cart)', 'ig');
 
-      if (elementText.length) {
-        // and these mean its out of stock
-        for (const outOfStockText of outOfStockTexts) {
-          if (elementText.includes(outOfStockText)) {
-            return elementText; // item is out of stock
-          }
+    // The out-of-stock or in-stock-text is generally always above-the-fold
+    // and often below-the-fold is a list of related products that may or may not contain trigger text
+    // so it's good to filter to just the 'above the fold' elements
+    // and it should be atleast 100px from the top to ignore items in the toolbar, sometimes menu items like "Coming soon" exist
+    const elementsToScan = Array.from(document.getElementsByTagName('*')).filter(element => element.getBoundingClientRect().top + window.scrollY <= vh && element.getBoundingClientRect().top + window.scrollY >= 100);
+
+    var elementText = "";
+
+    // REGEXS THAT REALLY MEAN IT'S IN STOCK
+    for (let i = elementsToScan.length - 1; i >= 0; i--) {
+        const element = elementsToScan[i];
+        elementText = "";
+        if (element.tagName.toLowerCase() === "input") {
+            elementText = element.value.toLowerCase();
+        } else {
+            elementText = getElementBaseText(element);
         }
-      }
-    }
-  }
 
-  return 'Possibly in stock'; // possibly in stock, cant decide otherwise.
+        if (elementText.length) {
+            // try which ones could mean its in stock
+            if (negateOutOfStockRegex.test(elementText)) {
+                return 'Possibly in stock';
+            }
+        }
+    }
+
+    // OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK
+    for (let i = elementsToScan.length - 1; i >= 0; i--) {
+        const element = elementsToScan[i];
+        if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) {
+            elementText = "";
+            if (element.tagName.toLowerCase() === "input") {
+                elementText = element.value.toLowerCase();
+            } else {
+                elementText = getElementBaseText(element);
+            }
+
+            if (elementText.length) {
+                // and these mean its out of stock
+                for (const outOfStockText of outOfStockTexts) {
+                    if (elementText.includes(outOfStockText)) {
+                        return outOfStockText; // item is out of stock
+                    }
+                }
+            }
+        }
+    }
+
+    return 'Possibly in stock'; // possibly in stock, cant decide otherwise.
 }
 
 // returns the element text that makes it think it's out of stock
-return isItemInStock();
+return isItemInStock().trim()
+

changedetectionio/run_custom_browser_url_tests.sh

@@ -6,16 +6,16 @@
 set -x
 
 # A extra browser is configured, but we never chose to use it, so it should NOT show in the logs
-docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_not_via_custom_browser_url'
-docker logs browserless-custom-url &>log.txt
+docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_not_via_custom_browser_url'
+docker logs sockpuppetbrowser-custom-url &>log.txt
 grep 'custom-browser-search-string=1' log.txt
 if [ $? -ne 1 ]
 then
-  echo "Saw a request in 'browserless-custom-url' container with 'custom-browser-search-string=1' when I should not"
+  echo "Saw a request in 'sockpuppetbrowser-custom-url' container with 'custom-browser-search-string=1' when I should not"
   exit 1
 fi
 
-docker logs browserless &>log.txt
+docker logs sockpuppetbrowser &>log.txt
 grep 'custom-browser-search-string=1' log.txt
 if [ $? -ne 1 ]
 then
@@ -24,16 +24,16 @@ then
 fi
 
 # Special connect string should appear in the custom-url container, but not in the 'default' one
-docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_via_custom_browser_url'
-docker logs browserless-custom-url &>log.txt
+docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_via_custom_browser_url'
+docker logs sockpuppetbrowser-custom-url &>log.txt
 grep 'custom-browser-search-string=1' log.txt
 if [ $? -ne 0 ]
 then
-  echo "Did not see request in 'browserless-custom-url' container with 'custom-browser-search-string=1' when I should"
+  echo "Did not see request in 'sockpuppetbrowser-custom-url' container with 'custom-browser-search-string=1' when I should"
   exit 1
 fi
 
-docker logs browserless &>log.txt
+docker logs sockpuppetbrowser &>log.txt
 grep 'custom-browser-search-string=1' log.txt
 if [ $? -ne 1 ]
 then

changedetectionio/run_proxy_tests.sh

@@ -35,7 +35,7 @@ docker run --network changedet-network \
 docker run --network changedet-network \
   -e "SOCKSTEST=manual-playwright" \
   -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/app/changedetectionio/test-datastore/proxies.json \
-  -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" \
+  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" \
   --rm \
   test-changedetectionio \
   bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy_sources.py'

changedetectionio/static/images/steps.svg

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   aria-hidden="true"
+   viewBox="0 0 19.966091 17.999964"
+   class="css-1oqmxjn"
+   version="1.1"
+   id="svg4"
+   sodipodi:docname="steps.svg"
+   width="19.966091"
+   height="17.999964"
+   inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <defs
+     id="defs8" />
+  <sodipodi:namedview
+     id="namedview6"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     showgrid="false"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0"
+     inkscape:zoom="8.6354167"
+     inkscape:cx="-1.3896261"
+     inkscape:cy="6.1375151"
+     inkscape:window-width="1280"
+     inkscape:window-height="667"
+     inkscape:window-x="2419"
+     inkscape:window-y="250"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg4" />
+  <path
+     d="m 16.95807,12.000003 c -0.7076,0.0019 -1.3917,0.2538 -1.9316,0.7113 -0.5398,0.4575 -0.9005,1.091 -1.0184,1.7887 H 5.60804 c -0.80847,0.0297 -1.60693,-0.1865 -2.29,-0.62 -0.26632,-0.1847 -0.48375,-0.4315 -0.63356,-0.7189 -0.14982,-0.2874 -0.22753,-0.607 -0.22644,-0.9311 -0.02843,-0.3931 0.03646,-0.7873 0.1894,-1.1505 0.15293,-0.3632 0.38957,-0.6851 0.6906,-0.9395 0.66628,-0.4559004 1.4637,-0.6807004 2.27,-0.6400004 h 8.35003 c 0.8515,-0.0223 1.6727,-0.3206 2.34,-0.85 0.3971,-0.3622 0.7076,-0.8091 0.9084,-1.3077 0.2008,-0.49857 0.2868,-1.03596 0.2516,-1.57229 0.0113,-0.47161 -0.0887,-0.93924 -0.292,-1.36493 -0.2033,-0.4257 -0.5041,-0.79745 -0.878,-1.08507 -0.7801,-0.55815 -1.7212,-0.84609 -2.68,-0.82 H 5.95804 c -0.12537,-0.7417 -0.5248,-1.40924 -1.11913,-1.87032996 -0.59434,-0.46108 -1.3402,-0.68207 -2.08979,-0.61917 -0.74958,0.06291 -1.44818,0.40512 -1.95736,0.95881 C 0.28259,1.5230126 0,2.2477926 0,3.0000126 c 0,0.75222 0.28259,1.47699 0.79176,2.03068 0.50918,0.55369 1.20778,0.8959 1.95736,0.95881 0.74959,0.0629 1.49545,-0.15808 2.08979,-0.61917 0.59433,-0.46109 0.99376,-1.12863 1.11913,-1.87032 h 7.70003 c 0.7353,-0.03061 1.4599,0.18397 2.06,0.61 0.2548,0.19335 0.4595,0.445 0.597,0.73385 0.1375,0.28884 0.2036,0.60644 0.193,0.92615 0.0316,0.38842 -0.0247,0.77898 -0.165,1.14258 -0.1402,0.36361 -0.3607,0.69091 -0.645,0.95741 -0.5713,0.4398 -1.2799,0.663 -2,0.63 H 5.69804 c -1.03259,-0.0462 -2.05065,0.2568 -2.89,0.86 -0.43755,0.3361 -0.78838,0.7720004 -1.02322,1.2712004 -0.23484,0.4993 -0.34688,1.0474 -0.32678,1.5988 -0.00726,0.484 0.10591,0.9622 0.32934,1.3916 0.22344,0.4295 0.55012,0.7966 0.95066,1.0684 0.85039,0.5592 1.85274,0.8421 2.87,0.81 h 8.40003 c 0.0954,0.5643 0.3502,1.0896 0.7343,1.5138 0.3842,0.4242 0.8817,0.7297 1.4338,0.8803 0.5521,0.1507 1.1358,0.1403 1.6822,-0.0299 0.5464,-0.1702 1.0328,-0.4932 1.4016,-0.9308 0.3688,-0.4376 0.6048,-0.9716 0.6801,-1.5389 0.0752,-0.5673 -0.0134,-1.1444 -0.2554,-1.663 -0.242,-0.5186 -0.6273,-0.9572 -1.1104,-1.264 -0.4831,-0.3068 -1.0439,-0.469 -1.6162,-0.4675 z m 0,5 c -0.3956,0 -0.7823,-0.1173 -1.1112,-0.3371 -0.3289,-0.2197 -0.5852,-0.5321 -0.7366,-0.8975 -0.1514,-0.3655 -0.191,-0.7676 -0.1138,-1.1556 0.0772,-0.3879 0.2677,-0.7443 0.5474,-1.024 0.2797,-0.2797 0.636,-0.4702 1.024,-0.5474 0.388,-0.0771 0.7901,-0.0375 1.1555,0.1138 0.3655,0.1514 0.6778,0.4078 0.8976,0.7367 0.2198,0.3289 0.3371,0.7155 0.3371,1.1111 0,0.5304 -0.2107,1.0391 -0.5858,1.4142 -0.3751,0.3751 -0.8838,0.5858 -1.4142,0.5858 z"
+     id="path2"
+     style="fill:#777777;fill-opacity:1" />
+</svg>

changedetectionio/static/js/browser-steps.js

@@ -10,7 +10,7 @@ $(document).ready(function () {
         }
     })
     var browsersteps_session_id;
-    var browserless_seconds_remaining = 0;
+    var browser_interface_seconds_remaining = 0;
     var apply_buttons_disabled = false;
     var include_text_elements = $("#include_text_elements");
     var xpath_data = false;
@@ -49,7 +49,7 @@ $(document).ready(function () {
         $('#browsersteps-img').removeAttr('src');
         $("#browsersteps-click-start").show();
         $("#browsersteps-selector-wrapper .spinner").hide();
-        browserless_seconds_remaining = 0;
+        browser_interface_seconds_remaining = 0;
         browsersteps_session_id = false;
         apply_buttons_disabled = false;
         ctx.clearRect(0, 0, c.width, c.height);
@@ -61,12 +61,12 @@ $(document).ready(function () {
         $('#browser_steps >li:first-child').css('opacity', '0.5');
     }
 
-    // Show seconds remaining until playwright/browserless needs to restart the session
+    // Show seconds remaining until the browser interface needs to restart the session
     // (See comment at the top of changedetectionio/blueprint/browser_steps/__init__.py )
     setInterval(() => {
-        if (browserless_seconds_remaining >= 1) {
-            document.getElementById('browserless-seconds-remaining').innerText = browserless_seconds_remaining + " seconds remaining in session";
-            browserless_seconds_remaining -= 1;
+        if (browser_interface_seconds_remaining >= 1) {
+            document.getElementById('browser-seconds-remaining').innerText = browser_interface_seconds_remaining + " seconds remaining in session";
+            browser_interface_seconds_remaining -= 1;
         }
     }, "1000")
 
@@ -261,7 +261,7 @@ $(document).ready(function () {
             // This should trigger 'Goto site'
             console.log("Got startup response, requesting Goto-Site (first) step fake click");
             $('#browser_steps >li:first-child .apply').click();
-            browserless_seconds_remaining = 500;
+            browser_interface_seconds_remaining = 500;
             set_first_gotosite_disabled();
         }).fail(function (data) {
             console.log(data);

changedetectionio/static/js/diff-overview.js

@@ -90,5 +90,10 @@ $(document).ready(function () {
         }
     }
 
-
+    $('#diff-form').on('submit', function (e) {
+        if ($('select[name=from_version]').val() === $('select[name=to_version]').val()) {
+            e.preventDefault();
+            alert('Error - You are trying to compare the same version.');
+        }
+    });
 });

changedetectionio/static/styles/diff.css

@@ -126,6 +126,8 @@ html[data-darkmode="true"] {
   html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
   html[data-darkmode="true"] .watch-table .current-diff-url::after {
     filter: invert(0.5) hue-rotate(10deg) brightness(2); }
+  html[data-darkmode="true"] .watch-table .status-browsersteps {
+    filter: invert(0.5) hue-rotate(10deg) brightness(1.5); }
   html[data-darkmode="true"] .watch-table .watch-controls .state-off img {
     opacity: 0.3; }
   html[data-darkmode="true"] .watch-table .watch-controls .state-on img {

changedetectionio/static/styles/scss/parts/_variables.scss

@@ -152,6 +152,10 @@ html[data-darkmode="true"] {
       filter: invert(.5) hue-rotate(10deg) brightness(2);
     }
 
+    .status-browsersteps {
+      filter: invert(.5) hue-rotate(10deg) brightness(1.5);
+    }
+
     .watch-controls {
       .state-off {
         img {

changedetectionio/static/styles/styles.css

@@ -342,6 +342,8 @@ html[data-darkmode="true"] {
   html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
   html[data-darkmode="true"] .watch-table .current-diff-url::after {
     filter: invert(0.5) hue-rotate(10deg) brightness(2); }
+  html[data-darkmode="true"] .watch-table .status-browsersteps {
+    filter: invert(0.5) hue-rotate(10deg) brightness(1.5); }
   html[data-darkmode="true"] .watch-table .watch-controls .state-off img {
     opacity: 0.3; }
   html[data-darkmode="true"] .watch-table .watch-controls .state-on img {

changedetectionio/store.py

@@ -9,7 +9,6 @@ from copy import deepcopy, copy
 from os import path, unlink
 from threading import Lock
 import json
-import logging
 import os
 import re
 import requests
@@ -17,6 +16,7 @@ import secrets
 import threading
 import time
 import uuid as uuid_builder
+from loguru import logger
 
 # Because the server will run as a daemon and wont know the URL for notification links when firing off a notification
 BASE_URL_NOT_SET_TEXT = '("Base URL" not set - see settings - notifications)'
@@ -42,7 +42,7 @@ class ChangeDetectionStore:
         self.__data = App.model()
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
-        print(">>> Datastore path is ", self.json_store_path)
+        logger.info(f"Datastore path is '{self.json_store_path}'")
         self.needs_write = False
         self.start_time = time.time()
         self.stop_thread = False
@@ -83,12 +83,12 @@ class ChangeDetectionStore:
                 for uuid, watch in self.__data['watching'].items():
                     watch['uuid']=uuid
                     self.__data['watching'][uuid] = Watch.model(datastore_path=self.datastore_path, default=watch)
-                    print("Watching:", uuid, self.__data['watching'][uuid]['url'])
+                    logger.info(f"Watching: {uuid} {self.__data['watching'][uuid]['url']}")
 
         # First time ran, Create the datastore.
         except (FileNotFoundError):
             if include_default_watches:
-                print("No JSON DB found at {}, creating JSON store at {}".format(self.json_store_path, self.datastore_path))
+                logger.critical(f"No JSON DB found at {self.json_store_path}, creating JSON store at {self.datastore_path}")
                 self.add_watch(url='https://news.ycombinator.com/',
                                tag='Tech news',
                                extras={'fetch_backend': 'html_requests'})
@@ -139,7 +139,7 @@ class ChangeDetectionStore:
         save_data_thread = threading.Thread(target=self.save_datastore).start()
 
     def set_last_viewed(self, uuid, timestamp):
-        logging.debug("Setting watch UUID: {} last viewed to {}".format(uuid, int(timestamp)))
+        logger.debug(f"Setting watch UUID: {uuid} last viewed to {int(timestamp)}")
         self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
         self.needs_write = True
 
@@ -255,6 +255,7 @@ class ChangeDetectionStore:
                 'last_viewed': 0,
                 'previous_md5': False,
                 'previous_md5_before_filters': False,
+                'remote_server_reply': None,
                 'track_ldjson_price_data': None,
             })
 
@@ -316,7 +317,7 @@ class ChangeDetectionStore:
                             apply_extras['include_filters'] = [res['css_filter']]
 
             except Exception as e:
-                logging.error("Error fetching metadata for shared watch link", url, str(e))
+                logger.error(f"Error fetching metadata for shared watch link {url} {str(e)}")
                 flash("Error fetching metadata for {}".format(url), 'error')
                 return False
         from .model.Watch import is_safe_url
@@ -345,7 +346,7 @@ class ChangeDetectionStore:
 
         new_uuid = new_watch.get('uuid')
 
-        logging.debug("Added URL {} - {}".format(url, new_uuid))
+        logger.debug(f"Adding URL {url} - {new_uuid}")
 
         for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
             if k in apply_extras:
@@ -362,7 +363,7 @@ class ChangeDetectionStore:
         if write_to_disk_now:
             self.sync_to_json()
 
-        print("added ", url)
+        logger.debug(f"Added '{url}'")
 
         return new_uuid
 
@@ -416,14 +417,13 @@ class ChangeDetectionStore:
 
 
     def sync_to_json(self):
-        logging.info("Saving JSON..")
-        print("Saving JSON..")
+        logger.info("Saving JSON..")
         try:
             data = deepcopy(self.__data)
         except RuntimeError as e:
             # Try again in 15 seconds
             time.sleep(15)
-            logging.error ("! Data changed when writing to JSON, trying again.. %s", str(e))
+            logger.error(f"! Data changed when writing to JSON, trying again.. {str(e)}")
             self.sync_to_json()
             return
         else:
@@ -436,7 +436,7 @@ class ChangeDetectionStore:
                     json.dump(data, json_file, indent=4)
                 os.replace(self.json_store_path+".tmp", self.json_store_path)
             except Exception as e:
-                logging.error("Error writing JSON!! (Main JSON file save was skipped) : %s", str(e))
+                logger.error(f"Error writing JSON!! (Main JSON file save was skipped) : {str(e)}")
 
             self.needs_write = False
             self.needs_write_urgent = False
@@ -447,7 +447,16 @@ class ChangeDetectionStore:
 
         while True:
             if self.stop_thread:
-                print("Shutting down datastore thread")
+                # Suppressing "Logging error in Loguru Handler #0" during CICD.
+                # Not a meaningful difference for a real use-case just for CICD.
+                # the side effect is a "Shutting down datastore thread" message
+                # at the end of each test.
+                # But still more looking better.
+                import sys
+                logger.remove()
+                logger.add(sys.stderr)
+
+                logger.critical("Shutting down datastore thread")
                 return
 
             if self.needs_write or self.needs_write_urgent:
@@ -463,7 +472,7 @@ class ChangeDetectionStore:
     # Go through the datastore path and remove any snapshots that are not mentioned in the index
     # This usually is not used, but can be handy.
     def remove_unused_snapshots(self):
-        print ("Removing snapshots from datastore that are not in the index..")
+        logger.info("Removing snapshots from datastore that are not in the index..")
 
         index=[]
         for uuid in self.data['watching']:
@@ -476,7 +485,7 @@ class ChangeDetectionStore:
         for uuid in self.data['watching']:
             for item in pathlib.Path(self.datastore_path).rglob(uuid+"/*.txt"):
                 if not str(item) in index:
-                    print ("Removing",item)
+                    logger.info(f"Removing {item}")
                     unlink(item)
 
     @property
@@ -562,7 +571,7 @@ class ChangeDetectionStore:
             if os.path.isfile(filepath):
                 headers.update(parse_headers_from_text_file(filepath))
         except Exception as e:
-            print(f"ERROR reading headers.txt at {filepath}", str(e))
+            logger.error(f"ERROR reading headers.txt at {filepath} {str(e)}")
 
         watch = self.data['watching'].get(uuid)
         if watch:
@@ -573,7 +582,7 @@ class ChangeDetectionStore:
                 if os.path.isfile(filepath):
                     headers.update(parse_headers_from_text_file(filepath))
             except Exception as e:
-                print(f"ERROR reading headers.txt at {filepath}", str(e))
+                logger.error(f"ERROR reading headers.txt at {filepath} {str(e)}")
 
             # In /datastore/tag-name.txt
             tags = self.get_all_tags_for_watch(uuid=uuid)
@@ -584,7 +593,7 @@ class ChangeDetectionStore:
                     if os.path.isfile(filepath):
                         headers.update(parse_headers_from_text_file(filepath))
                 except Exception as e:
-                    print(f"ERROR reading headers.txt at {filepath}", str(e))
+                    logger.error(f"ERROR reading headers.txt at {filepath} {str(e)}")
 
         return headers
 
@@ -602,13 +611,13 @@ class ChangeDetectionStore:
     def add_tag(self, name):
         # If name exists, return that
         n = name.strip().lower()
-        print (f">>> Adding new tag - '{n}'")
+        logger.debug(f">>> Adding new tag - '{n}'")
         if not n:
             return False
 
         for uuid, tag in self.__data['settings']['application'].get('tags', {}).items():
             if n == tag.get('title', '').lower().strip():
-                print (f">>> Tag {name} already exists")
+                logger.warning(f"Tag '{name}' already exists, skipping creation.")
                 return uuid
 
         # Eventually almost everything todo with a watch will apply as a Tag
@@ -670,7 +679,7 @@ class ChangeDetectionStore:
         updates_available = self.get_updates_available()
         for update_n in updates_available:
             if update_n > self.__data['settings']['application']['schema_version']:
-                print ("Applying update_{}".format((update_n)))
+                logger.critical(f"Applying update_{update_n}")
                 # Wont exist on fresh installs
                 if os.path.exists(self.json_store_path):
                     shutil.copyfile(self.json_store_path, self.datastore_path+"/url-watches-before-{}.json".format(update_n))
@@ -678,8 +687,8 @@ class ChangeDetectionStore:
                 try:
                     update_method = getattr(self, "update_{}".format(update_n))()
                 except Exception as e:
-                    print("Error while trying update_{}".format((update_n)))
-                    print(e)
+                    logger.error(f"Error while trying update_{update_n}")
+                    logger.error(e)
                     # Don't run any more updates
                     return
                 else:
@@ -717,7 +726,7 @@ class ChangeDetectionStore:
                         with open(os.path.join(target_path, "history.txt"), "w") as f:
                             f.writelines(history)
                     else:
-                        logging.warning("Datastore history directory {} does not exist, skipping history import.".format(target_path))
+                        logger.warning(f"Datastore history directory {target_path} does not exist, skipping history import.")
 
                 # No longer needed, dynamically pulled from the disk when needed.
                 # But we should set it back to a empty dict so we don't break if this schema runs on an earlier version.

changedetectionio/templates/_common_fields.jinja

@@ -115,6 +115,12 @@
 									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. <br>
                                     For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
                                     </p>
+                                    <p>
+                                        For JSON payloads, use <strong>|tojson</strong> without quotes for automatic escaping, for example - <code>{ "name": {{ '{{ watch_title|tojson }}' }} }</code>
+                                    </p>
+                                    <p>
+                                        URL encoding, use <strong>|urlencode</strong>, for example - <code>gets://hook-website.com/test.php?title={{ '{{ watch_title|urlencode }}' }}</code>
+                                    </p>
                                 </div>
                             </div>
                             <div class="pure-control-group">

changedetectionio/templates/diff.html

@@ -13,7 +13,7 @@
 <script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
 
 <div id="settings">
-    <form class="pure-form " action="" method="GET">
+    <form class="pure-form " action="" method="GET" id="diff-form">
         <fieldset>
             {% if versions|length >= 1 %}
                 <strong>Compare</strong>

changedetectionio/templates/edit.html

@@ -228,7 +228,7 @@ User-Agent: wonderbra 1.0") }}
                                 </div>
                             </div>
                             <div id="browser-steps-fieldlist" style="padding-left: 1em;  width: 350px; font-size: 80%;" >
-                                <span id="browserless-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target=_new href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
+                                <span id="browser-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target=_new href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
                                 {{ render_field(form.browser_steps) }}
                             </div>
                         </div>
@@ -339,6 +339,10 @@ nav
                     <span class="pure-form-message-inline">When content is merely moved in a list, it will also trigger an <strong>addition</strong>, consider enabling <code><strong>Only trigger when unique lines appear</strong></code></span>
                 </fieldset>
 
+                <fieldset class="pure-control-group">
+                    {{ render_checkbox_field(form.sort_text_alphabetically) }}
+                    <span class="pure-form-message-inline">Helps reduce changes detected caused by sites shuffling lines around, combine with <i>check unique lines</i> below.</span>
+                </fieldset>
                 <fieldset class="pure-control-group">
                     {{ render_checkbox_field(form.check_unique_lines) }}
                     <span class="pure-form-message-inline">Good for websites that just move the content around, and you want to know when NEW content is added, compares new lines against all history for this watch.</span>
@@ -401,6 +405,7 @@ Unavailable") }}
                                 <li>Use <code>//(?aiLmsux))</code> type flags (more <a href="https://docs.python.org/3/library/re.html#index-15">information here</a>)<br></li>
                                 <li>Keyword example &dash; example <code>Out of stock</code></li>
                                 <li>Use groups to extract just that text &dash; example <code>/reports.+?(\d+)/i</code> returns a list of years only</li>
+                                <li>Example - match lines containing a keyword <code>/.*icecream.*/</code></li>
                             </ul>
                         </li>
                         <li>One line per regular-expression/string match</li>
@@ -482,6 +487,10 @@ Unavailable") }}
                             <td>Last fetch time</td>
                             <td>{{ watch.fetch_time }}s</td>
                         </tr>
+                        <tr>
+                            <td>Notification alert count</td>
+                            <td>{{ watch.notification_alert_count }}</td>
+                        </tr>
                         </tbody>
                     </table>
                 </div>

changedetectionio/templates/watch-overview.html

@@ -110,6 +110,7 @@
                     {% endif %}
 
                     {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" >{% endif %}
+                    {% if watch.has_browser_steps %}<img class="status-icon status-browsersteps" src="{{url_for('static_content', group='images', filename='steps.svg')}}" title="Browser Steps is enabled" >{% endif %}
                     {% if watch.last_error is defined and watch.last_error != False %}
                     <div class="fetch-error">{{ watch.last_error }}
 

changedetectionio/tests/conftest.py

@@ -4,6 +4,8 @@ import pytest
 from changedetectionio import changedetection_app
 from changedetectionio import store
 import os
+import sys
+from loguru import logger
 
 # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py
 # Much better boilerplate than the docs
@@ -11,6 +13,15 @@ import os
 
 global app
 
+# https://loguru.readthedocs.io/en/latest/resources/migration.html#replacing-caplog-fixture-from-pytest-library
+# Show loguru logs only if CICD pytest fails.
+from loguru import logger
+@pytest.fixture
+def reportlog(pytestconfig):
+    logging_plugin = pytestconfig.pluginmanager.getplugin("logging-plugin")
+    handler_id = logger.add(logging_plugin.report_handler, format="{message}")
+    yield
+    logger.remove(handler_id)
 
 def cleanup(datastore_path):
     import glob
@@ -41,6 +52,18 @@ def app(request):
 
     app_config = {'datastore_path': datastore_path, 'disable_checkver' : True}
     cleanup(app_config['datastore_path'])
+
+    logger_level = 'TRACE'
+
+    logger.remove()
+    log_level_for_stdout = { 'DEBUG', 'SUCCESS' }
+    logger.configure(handlers=[
+        {"sink": sys.stdout, "level": logger_level,
+         "filter" : lambda record: record['level'].name in log_level_for_stdout},
+        {"sink": sys.stderr, "level": logger_level,
+         "filter": lambda record: record['level'].name not in log_level_for_stdout},
+        ])
+
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
     app = changedetection_app(app_config, datastore)
 

changedetectionio/tests/custom_browser_url/test_custom_browser_url.py

@@ -7,10 +7,11 @@ from ..util import live_server_setup, wait_for_all_checks
 def do_test(client, live_server, make_test_use_extra_browser=False):
 
     # Grep for this string in the logs?
-    test_url = f"https://changedetection.io/ci-test.html"
+    test_url = f"https://changedetection.io/ci-test.html?non-custom-default=true"
+    # "non-custom-default" should not appear in the custom browser connection
     custom_browser_name = 'custom browser URL'
 
-    # needs to be set and something like 'ws://127.0.0.1:3000?stealth=1&--disable-web-security=true'
+    # needs to be set and something like 'ws://127.0.0.1:3000'
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
 
     #####################
@@ -19,9 +20,7 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
         data={"application-empty_pages_are_a_change": "",
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_webdriver",
-              # browserless-custom-url is setup in  .github/workflows/test-only.yml
-              # the test script run_custom_browser_url_test.sh will look for 'custom-browser-search-string' in the container logs
-              'requests-extra_browsers-0-browser_connection_url': 'ws://browserless-custom-url:3000?stealth=1&--disable-web-security=true&custom-browser-search-string=1',
+              'requests-extra_browsers-0-browser_connection_url': 'ws://sockpuppetbrowser-custom-url:3000',
               'requests-extra_browsers-0-browser_name': custom_browser_name
               },
         follow_redirects=True
@@ -51,7 +50,8 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
         res = client.post(
             url_for("edit_page", uuid="first"),
             data={
-                  "url": test_url,
+                # 'run_customer_browser_url_tests.sh' will search for this string to know if we hit the right browser container or not
+                  "url": f"https://changedetection.io/ci-test.html?custom-browser-search-string=1",
                   "tags": "",
                   "headers": "",
                   'fetch_backend': f"extra_browser_{custom_browser_name}",

changedetectionio/tests/fetchers/test_content.py

@@ -37,4 +37,4 @@ def test_fetch_webdriver_content(client, live_server):
     )
     logging.getLogger().info("Looking for correct fetched HTML (text) from server")
 
-    assert b'cool it works' in res.data
+    assert b'cool it works' in res.data

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -97,6 +97,17 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     set_original_response()
     global smtp_test_server
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
+    notification_body = f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>My Webpage</title>
+</head>
+<body>
+    <h1>Test</h1>
+    {default_notification_body}
+</body>
+</html>
+"""
 
     #####################
     # Set this up for when we remove the notification from the watch, it should fallback with these details
@@ -104,7 +115,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
         url_for("settings_page"),
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": default_notification_body,
+              "application-notification_body": notification_body,
               "application-notification_format": 'Text',
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
@@ -161,5 +172,10 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     assert 'Content-Type: text/html' in msg
     assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
 
+    # https://github.com/dgtlmoon/changedetection.io/issues/2103
+    assert '<h1>Test</h1>' in msg
+    assert '&lt;' not in msg
+    assert 'Content-Type: text/html' in msg
+
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_api.py

@@ -163,6 +163,7 @@ def test_api_simple(client, live_server):
     # Loading the most recent snapshot should force viewed to become true
     client.get(url_for("diff_history_page", uuid="first"), follow_redirects=True)
 
+    time.sleep(3)
     # Fetch the whole watch again, viewed should be true
     res = client.get(
         url_for("watch", uuid=watch_uuid),

changedetectionio/tests/test_backup.py

@@ -1,8 +1,8 @@
 #!/usr/bin/python3
 
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+from .util import set_original_response, live_server_setup, wait_for_all_checks
 from flask import url_for
-from urllib.request import urlopen
+import io
 from zipfile import ZipFile
 import re
 import time
@@ -37,15 +37,10 @@ def test_backup(client, live_server):
     # Should be PK/ZIP stream
     assert res.data.count(b'PK') >= 2
 
-    # ZipFile from buffer seems non-obvious, just save it instead
-    with open("download.zip", 'wb') as f:
-        f.write(res.data)
-
-    zip = ZipFile('download.zip')
-    l = zip.namelist()
+    backup = ZipFile(io.BytesIO(res.data))
+    l = backup.namelist()
     uuid4hex = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}.*txt', re.I)
     newlist = list(filter(uuid4hex.match, l))  # Read Note below
 
     # Should be two txt files in the archive (history and the snapshot)
     assert len(newlist) == 2
-

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -456,7 +456,7 @@ def test_ignore_json_order(client, live_server):
 
 def test_correct_header_detect(client, live_server):
     # Like in https://github.com/dgtlmoon/changedetection.io/pull/1593
-    # Specify extra html that JSON is sometimes wrapped in - when using Browserless/Puppeteer etc
+    # Specify extra html that JSON is sometimes wrapped in - when using SockpuppetBrowser / Puppeteer / Playwrightetc
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write('<html><body>{"hello" : 123, "world": 123}')
 

changedetectionio/tests/test_notification_errors.py

@@ -1,8 +1,7 @@
 import os
 import time
-import re
 from flask import url_for
-from . util import set_original_response, set_modified_response, live_server_setup
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 import logging
 
 def test_check_notification_error_handling(client, live_server):
@@ -11,7 +10,7 @@ def test_check_notification_error_handling(client, live_server):
     set_original_response()
 
     # Give the endpoint time to spin up
-    time.sleep(2)
+    time.sleep(1)
 
     # Set a URL and fetch it, then set a notification URL which is going to give errors
     test_url = url_for('test_endpoint', _external=True)
@@ -22,12 +21,16 @@ def test_check_notification_error_handling(client, live_server):
     )
     assert b"Watch added" in res.data
 
-    time.sleep(2)
+    wait_for_all_checks(client)
     set_modified_response()
 
+    working_notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
+    broken_notification_url = "jsons://broken-url-xxxxxxxx123/test"
+
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"notification_urls": "jsons://broken-url-xxxxxxxx123/test",
+        # A URL with errors should not block the one that is working
+        data={"notification_urls": f"{broken_notification_url}\r\n{working_notification_url}",
               "notification_title": "xxx",
               "notification_body": "xxxxx",
               "notification_format": "Text",
@@ -63,4 +66,10 @@ def test_check_notification_error_handling(client, live_server):
     found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
     assert found_name_resolution_error
 
+    # And the working one, which is after the 'broken' one should still have fired
+    with open("test-datastore/notification.txt", "r") as f:
+        notification_submission = f.read()
+    os.unlink("test-datastore/notification.txt")
+    assert 'xxxxx' in notification_submission
+
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_request.py

@@ -10,11 +10,11 @@ def test_setup(live_server):
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
 def test_headers_in_request(client, live_server):
-    #live_server_setup(live_server)
+    #ve_server_setup(live_server)
     # Add our URL to the import page
     test_url = url_for('test_headers', _external=True)
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        # Because its no longer calling back to localhost but from browserless, set in test-only.yml
+        # Because its no longer calling back to localhost but from the browser container, set in test-only.yml
         test_url = test_url.replace('localhost', 'changedet')
 
     # Add the test URL twice, we will check
@@ -70,16 +70,17 @@ def test_headers_in_request(client, live_server):
 
     wait_for_all_checks(client)
 
-    # Re #137 -  Examine the JSON index file, it should have only one set of headers entered
+    # Re #137 -  It should have only one set of headers entered
     watches_with_headers = 0
-    with open('test-datastore/url-watches.json') as f:
-        app_struct = json.load(f)
-        for uuid in app_struct['watching']:
-            if (len(app_struct['watching'][uuid]['headers'])):
+    for k, watch in client.application.config.get('DATASTORE').data.get('watching').items():
+            if (len(watch['headers'])):
                 watches_with_headers += 1
+    assert watches_with_headers == 1
+
+    # 'server' http header was automatically recorded
+    for k, watch in client.application.config.get('DATASTORE').data.get('watching').items():
+        assert 'custom' in watch.get('remote_server_reply') # added in util.py
 
-    # Should be only one with headers set
-    assert watches_with_headers==1
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
@@ -88,7 +89,7 @@ def test_body_in_request(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_body', _external=True)
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        # Because its no longer calling back to localhost but from browserless, set in test-only.yml
+        # Because its no longer calling back to localhost but from the browser container, set in test-only.yml
         test_url = test_url.replace('localhost', 'cdio')
 
     res = client.post(
@@ -180,7 +181,7 @@ def test_method_in_request(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_method', _external=True)
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        # Because its no longer calling back to localhost but from browserless, set in test-only.yml
+        # Because its no longer calling back to localhost but from the browser container, set in test-only.yml
         test_url = test_url.replace('localhost', 'cdio')
 
     # Add the test URL twice, we will check
@@ -257,7 +258,7 @@ def test_headers_textfile_in_request(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_headers', _external=True)
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        # Because its no longer calling back to localhost but from browserless, set in test-only.yml
+        # Because its no longer calling back to localhost but from the browser container, set in test-only.yml
         test_url = test_url.replace('localhost', 'cdio')
 
     print ("TEST URL IS ",test_url)

changedetectionio/tests/test_unique_lines.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 
 
 def set_original_ignore_response():
@@ -34,6 +34,23 @@ def set_modified_swapped_lines():
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
+def set_modified_swapped_lines_with_extra_text_for_sorting():
+    test_return_data = """<html>
+     <body>
+     <p>&nbsp;Which is across multiple lines</p>     
+     <p>Some initial text</p>
+     <p>   So let's see what happens.</p>
+     <p>Z last</p>
+     <p>0 numerical</p>
+     <p>A uppercase</p>
+     <p>a lowercase</p>     
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
 
 def set_modified_with_trigger_text_response():
     test_return_data = """<html>
@@ -49,15 +66,14 @@ def set_modified_with_trigger_text_response():
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
-
-def test_unique_lines_functionality(client, live_server):
+def test_setup(client, live_server):
     live_server_setup(live_server)
 
-    sleep_time_for_fetch_thread = 3
+def test_unique_lines_functionality(client, live_server):
+    #live_server_setup(live_server)
+
 
     set_original_ignore_response()
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -67,7 +83,7 @@ def test_unique_lines_functionality(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # Add our URL to the import page
     res = client.post(
@@ -83,12 +99,11 @@ def test_unique_lines_functionality(client, live_server):
     #  Make a change
     set_modified_swapped_lines()
 
-    time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -97,7 +112,57 @@ def test_unique_lines_functionality(client, live_server):
     # Now set the content which contains the new text and re-ordered existing text
     set_modified_with_trigger_text_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
 
+def test_sort_lines_functionality(client, live_server):
+    #live_server_setup(live_server)
+
+    set_modified_swapped_lines_with_extra_text_for_sorting()
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"sort_text_alphabetically": "n",
+              "url": test_url,
+              "fetch_backend": "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    wait_for_all_checks(client)
+
+
+    res = client.get(url_for("index"))
+    # Should be a change registered
+    assert b'unviewed' in res.data
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert res.data.find(b'0 numerical') < res.data.find(b'Z last')
+    assert res.data.find(b'A uppercase') < res.data.find(b'Z last')
+    assert res.data.find(b'Some initial text') < res.data.find(b'Which is across multiple lines')
+    
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

changedetectionio/tests/util.py

@@ -175,12 +175,16 @@ def live_server_setup(live_server):
     @live_server.app.route('/test-headers')
     def test_headers():
 
-        output= []
+        output = []
 
         for header in request.headers:
-             output.append("{}:{}".format(str(header[0]),str(header[1])   ))
+            output.append("{}:{}".format(str(header[0]), str(header[1])))
 
-        return "\n".join(output)
+        content = "\n".join(output)
+
+        resp = make_response(content, 200)
+        resp.headers['server'] = 'custom'
+        return resp
 
     # Just return the body in the request
     @live_server.app.route('/test-body', methods=['POST', 'GET'])

changedetectionio/update_worker.py

@@ -12,14 +12,13 @@ from .processors.restock_diff import UnableToExtractRestockData
 # Requests for checking on a single site(watch) from a queue of watches
 # (another process inserts watches into the queue that are time-ready for checking)
 
-import logging
 import sys
+from loguru import logger
 
 class update_worker(threading.Thread):
     current_uuid = None
 
     def __init__(self, q, notification_q, app, datastore, *args, **kwargs):
-        logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
         self.q = q
         self.app = app
         self.notification_q = notification_q
@@ -32,6 +31,8 @@ class update_worker(threading.Thread):
         dates = []
         trigger_text = ''
 
+        now = time.time()
+
         if watch:
             watch_history = watch.history
             dates = list(watch_history.keys())
@@ -73,13 +74,14 @@ class update_worker(threading.Thread):
             'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep),
             'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
             'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
+            'notification_timestamp': now,
             'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
             'triggered_text': triggered_text,
             'uuid': watch.get('uuid') if watch else None,
             'watch_url': watch.get('url') if watch else None,
         })
-        logging.info(">> SENDING NOTIFICATION")
-
+        logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
+        logger.debug("Queued notification for sending")
         notification_q.put(n_object)
 
     # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
@@ -148,6 +150,10 @@ class update_worker(threading.Thread):
         queued = False
         if n_object and n_object.get('notification_urls'):
             queued = True
+
+            count = watch.get('notification_alert_count', 0) + 1
+            self.datastore.update_watch(uuid=watch_uuid, update_obj={'notification_alert_count': count})
+
             self.queue_notification_for_watch(notification_q=self.notification_q, n_object=n_object, watch=watch)
 
         return queued
@@ -180,7 +186,7 @@ class update_worker(threading.Thread):
                 'screenshot': None
             })
             self.notification_q.put(n_object)
-            print("Sent filter not found notification for {}".format(watch_uuid))
+            logger.error(f"Sent filter not found notification for {watch_uuid}")
 
     def send_step_failure_notification(self, watch_uuid, step_n):
         watch = self.datastore.data['watching'].get(watch_uuid, False)
@@ -207,7 +213,7 @@ class update_worker(threading.Thread):
                 'uuid': watch_uuid
             })
             self.notification_q.put(n_object)
-            print("Sent step not found notification for {}".format(watch_uuid))
+            logger.error(f"Sent step not found notification for {watch_uuid}")
 
 
     def cleanup_error_artifacts(self, uuid):
@@ -221,7 +227,8 @@ class update_worker(threading.Thread):
     def run(self):
 
         from .processors import text_json_diff, restock_diff
-
+        now = time.time()
+        
         while not self.app.config.exit.is_set():
             update_handler = None
 
@@ -233,14 +240,14 @@ class update_worker(threading.Thread):
             else:
                 uuid = queued_item_data.item.get('uuid')
                 self.current_uuid = uuid
-
                 if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
                     changed_detected = False
                     contents = b''
                     process_changedetection_results = True
                     update_obj = {}
-                    print("> Processing UUID {} Priority {} URL {}".format(uuid, queued_item_data.priority,
-                                                                           self.datastore.data['watching'][uuid]['url']))
+                    logger.info(f"Processing watch UUID {uuid} "
+                            f"Priority {queued_item_data.priority} "
+                            f"URL {self.datastore.data['watching'][uuid]['url']}")
                     now = time.time()
 
                     try:
@@ -280,7 +287,8 @@ class update_worker(threading.Thread):
                         if not isinstance(contents, (bytes, bytearray)):
                             raise Exception("Error - returned data from the fetch handler SHOULD be bytes")
                     except PermissionError as e:
-                        self.app.logger.error("File permission error updating", uuid, str(e))
+                        logger.critical(f"File permission error updating file, watch: {uuid}")
+                        logger.critical(str(e))
                         process_changedetection_results = False
                     except content_fetcher.ReplyWithContentButNoText as e:
                         # Totally fine, it's by choice - just continue on, nothing more to care about
@@ -338,7 +346,7 @@ class update_worker(threading.Thread):
                             # Send notification if we reached the threshold?
                             threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
                                                                                            0)
-                            print("Filter for {} not found, consecutive_filter_failures: {}".format(uuid, c))
+                            logger.error(f"Filter for {uuid} not found, consecutive_filter_failures: {c}")
                             if threshold > 0 and c >= threshold:
                                 if not self.datastore.data['watching'][uuid].get('notification_muted'):
                                     self.send_filter_failure_notification(uuid)
@@ -372,7 +380,7 @@ class update_worker(threading.Thread):
                             # Other Error, more info is good.
                             err_text += " " + str(e.original_e).splitlines()[0]
 
-                        print(f"BrowserSteps exception at step {error_step}", str(e.original_e))
+                        logger.debug(f"BrowserSteps exception at step {error_step} {str(e.original_e)}")
 
                         self.datastore.update_watch(uuid=uuid,
                                                     update_obj={'last_error': err_text,
@@ -386,7 +394,7 @@ class update_worker(threading.Thread):
                             # Send notification if we reached the threshold?
                             threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
                                                                                            0)
-                            print("Step for {} not found, consecutive_filter_failures: {}".format(uuid, c))
+                            logger.error(f"Step for {uuid} not found, consecutive_filter_failures: {c}")
                             if threshold > 0 and c >= threshold:
                                 if not self.datastore.data['watching'][uuid].get('notification_muted'):
                                     self.send_step_failure_notification(watch_uuid=uuid, step_n=e.step_n)
@@ -403,7 +411,7 @@ class update_worker(threading.Thread):
                                                                            'last_check_status': e.status_code})
                         process_changedetection_results = False
                     except content_fetcher.ScreenshotUnavailable as e:
-                        err_text = "Screenshot unavailable, page did not render fully in the expected time - try increasing 'Wait seconds before extracting text'"
+                        err_text = "Screenshot unavailable, page did not render fully in the expected time or page was too long - try increasing 'Wait seconds before extracting text'"
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
                         process_changedetection_results = False
@@ -426,13 +434,21 @@ class update_worker(threading.Thread):
                                                                            'last_check_status': e.status_code,
                                                                            'has_ldjson_price_data': None})
                         process_changedetection_results = False
+                    except content_fetcher.BrowserStepsInUnsupportedFetcher as e:
+                        err_text = "This watch has Browser Steps configured and so it cannot run with the 'Basic fast Plaintext/HTTP Client', either remove the Browser Steps or select a Chrome fetcher."
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
+                        process_changedetection_results = False
+                        logger.error(f"Exception (BrowserStepsInUnsupportedFetcher) reached processing watch UUID: {uuid}")
+
                     except UnableToExtractRestockData as e:
                         # Usually when fetcher.instock_data returns empty
-                        self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
+                        logger.error(f"Exception (UnableToExtractRestockData) reached processing watch UUID: {uuid}")
+                        logger.error(str(e))
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': f"Unable to extract restock data for this page unfortunately. (Got code {e.status_code} from server)"})
                         process_changedetection_results = False
                     except Exception as e:
-                        self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
+                        logger.error(f"Exception reached processing watch UUID: {uuid}")
+                        logger.error(str(e))
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
                         # Other serious error
                         process_changedetection_results = False
@@ -468,23 +484,33 @@ class update_worker(threading.Thread):
 
                             # A change was detected
                             if changed_detected:
-                                print (">> Change detected in UUID {} - {}".format(uuid, watch['url']))
-
                                 # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
                                 if watch.history_n >= 2:
+                                    logger.info(f"Change detected in UUID {uuid} - {watch['url']}")
                                     if not self.datastore.data['watching'][uuid].get('notification_muted'):
                                         self.send_content_changed_notification(watch_uuid=uuid)
-
+                                else:
+                                    logger.info(f"Change triggered in UUID {uuid} due to first history saving (no notifications sent) - {watch['url']}")
 
                         except Exception as e:
                             # Catch everything possible here, so that if a worker crashes, we don't lose it until restart!
-                            print("!!!! Exception in update_worker !!!\n", e)
-                            self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
+                            logger.critical("!!!! Exception in update_worker while processing process_changedetection_results !!!")
+                            logger.critical(str(e))
                             self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
 
                     if self.datastore.data['watching'].get(uuid):
                         # Always record that we atleast tried
                         count = self.datastore.data['watching'][uuid].get('check_count', 0) + 1
+
+                        # Record the 'server' header reply, can be used for actions in the future like cloudflare/akamai workarounds
+                        try:
+                            server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]
+                            self.datastore.update_watch(uuid=uuid,
+                                                        update_obj={'remote_server_reply': server_header}
+                                                        )
+                        except Exception as e:
+                            pass
+
                         self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
                                                                            'last_checked': round(time.time()),
                                                                            'check_count': count
@@ -499,6 +525,7 @@ class update_worker(threading.Thread):
 
                 self.current_uuid = None  # Done
                 self.q.task_done()
+                logger.debug(f"Watch {uuid} done in {time.time()-now:.2f}s")
 
                 # Give the CPU time to interrupt
                 time.sleep(0.1)

docker-compose.yml

@@ -16,6 +16,10 @@ services:
   #      - PUID=1000
   #      - PGID=1000
   #
+  #        Log levels are in descending order. (TRACE is the most detailed one)
+  #        Log output levels: TRACE, DEBUG(default), INFO, SUCCESS, WARNING, ERROR, CRITICAL
+  #      - LOGGER_LEVEL=DEBUG
+  #
   #       Alternative WebDriver/selenium URL, do not use "'s or 's!
   #      - WEBDRIVER_URL=http://browser-chrome:4444/wd/hub
   #
@@ -26,7 +30,7 @@ services:
   #             https://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.proxy
   #
   #       Alternative Playwright URL, do not use "'s or 's!
-  #      - PLAYWRIGHT_DRIVER_URL=ws://playwright-chrome:3000/?stealth=1&--disable-web-security=true
+  #      - PLAYWRIGHT_DRIVER_URL=ws://playwright-chrome:3000
   #
   #       Playwright proxy settings playwright_proxy_server, playwright_proxy_bypass, playwright_proxy_username, playwright_proxy_password
   #
@@ -67,30 +71,23 @@ services:
 #            condition: service_started
 
      # Used for fetching pages via Playwright+Chrome where you need Javascript support.
-     # Note: Playwright/browserless not supported on ARM type devices (rPi etc)
      # RECOMMENDED FOR FETCHING PAGES WITH CHROME
 #    playwright-chrome:
 #        hostname: playwright-chrome
-#        image: browserless/chrome:1.60-chrome-stable
+#        image: dgtlmoon/sockpuppetbrowser:latest
+#        cap_add:
+#            - SYS_ADMIN
+## SYS_ADMIN might be too much, but it can be needed on your platform https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#running-puppeteer-on-gitlabci
 #        restart: unless-stopped
 #        environment:
 #            - SCREEN_WIDTH=1920
 #            - SCREEN_HEIGHT=1024
 #            - SCREEN_DEPTH=16
-#            - ENABLE_DEBUGGER=false
-#            - PREBOOT_CHROME=true
-#            - CONNECTION_TIMEOUT=300000
-#            - MAX_CONCURRENT_SESSIONS=10
-#            - CHROME_REFRESH_TIME=600000
-#            - DEFAULT_BLOCK_ADS=true
-#            - DEFAULT_STEALTH=true
-#
-#             Ignore HTTPS errors, like for self-signed certs
-#            - DEFAULT_IGNORE_HTTPS_ERRORS=true
-#
+#            - MAX_CONCURRENT_CHROME_PROCESSES=10
 
      # Used for fetching pages via Playwright+Chrome where you need Javascript support.
-     # Note: works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector) and other issues
+     # Note: Works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector)
+     #       Does not report status codes (200, 404, 403) and other issues
 #    browser-chrome:
 #        hostname: browser-chrome
 #        image: selenium/standalone-chrome:4

requirements.txt

@@ -46,8 +46,8 @@ beautifulsoup4
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
 lxml
 
-# XPath 2.0-3.1 support
-elementpath
+# XPath 2.0-3.1 support - 4.2.0 broke something?
+elementpath==4.1.5
 
 selenium~=4.14.0
 
@@ -72,3 +72,5 @@ pytest-flask ~=1.2
 
 # Pin jsonschema version to prevent build errors on armv6 while rpds-py wheels aren't available (1708)
 jsonschema==4.17.3
+
+loguru