Re #2014 - When any of a list of notifications fails, the others should still work

.github/workflows/pypi-release.yml

@@ -0,0 +1,72 @@
+name: Publish Python 🐍distribution 📦 to PyPI and TestPyPI
+
+on: push
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
+    - name: Install pypa/build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: python3 -m build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+
+
+  test-pypi-package:
+    name: Test the built 📦 package works basically.
+    runs-on: ubuntu-latest
+    needs:
+    - build
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Test that the basic pip built package runs without error
+      run: |
+        set -e
+        pip3 install dist/changedetection.io*.whl
+        changedetection.io -d /tmp -p 10000 &
+        sleep 3
+        curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
+        curl http://127.0.0.1:10000/ >/dev/null
+        killall changedetection.io
+
+
+  publish-to-pypi:
+    name: >-
+      Publish Python 🐍 distribution 📦 to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - test-pypi-package
+    runs-on: ubuntu-latest
+    environment:
+      name: release
+      url: https://pypi.org/p/changedetection.io
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/test-only.yml

@@ -118,7 +118,8 @@ jobs:
           sleep 3
           # invert the check (it should be not 0/not running)
           docker ps
-          docker logs sig-test
+          # check signal catch(STDOUT) log
+          docker logs sig-test | grep 'Shutdown: Got Signal - SIGINT' || exit 1
           test -z "`docker ps|grep sig-test`"
           if [ $? -ne 0 ]
           then
@@ -138,7 +139,7 @@ jobs:
           sleep 3
           # invert the check (it should be not 0/not running)
           docker ps
-          docker logs sig-test
+          docker logs sig-test | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
           test -z "`docker ps|grep sig-test`"
           if [ $? -ne 0 ]
           then

.github/workflows/test-pip-build.yml

@@ -1,36 +0,0 @@
-name: ChangeDetection.io PIP package test
-
-# Triggers the workflow on push or pull request events
-
-# This line doesnt work, even tho it is the documented one
-on: [push, pull_request]
-
-  # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
-  # @todo: some kind of path filter for requirements.txt and Dockerfile
-jobs:
-  test-pip-build-basics:
-    runs-on: ubuntu-latest
-    steps:
-        - uses: actions/checkout@v4
-
-        - name: Set up Python 3.11
-          uses: actions/setup-python@v4
-          with:
-            python-version: 3.11
-
-
-        - name: Test that the basic pip built package runs without error
-          run: |
-            set -e
-            mkdir dist
-            pip3 install wheel
-            python3 setup.py bdist_wheel            
-            pip3 install -r requirements.txt
-            rm ./changedetection.py
-            rm -rf changedetectio
-            
-            pip3 install dist/changedetection.io*.whl
-            changedetection.io -d /tmp -p 10000 &
-            sleep 3
-            curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
-            killall -9 changedetection.io

MANIFEST.in

@@ -10,6 +10,8 @@ prune changedetectionio/static/package-lock.json
 prune changedetectionio/static/styles/node_modules
 prune changedetectionio/static/styles/package-lock.json
 include changedetection.py
+include requirements.txt
+include README-pip.md
 global-exclude *.pyc
 global-exclude node_modules
 global-exclude venv

Procfile

@@ -1 +0,0 @@
-web: python3 ./changedetection.py -C -d ./datastore -p $PORT

README.md

@@ -17,7 +17,7 @@ _Live your data-life pro-actively._
 - Nothing to install, access via browser login after signup.
 - Super fast, no registration needed setup.
 - Get started watching and receiving website change notifications straight away.
-
+- See our [tutorials and how-to page for more inspiration](https://changedetection.io/tutorials) 
 
 ### Target specific parts of the webpage using the Visual Selector tool.
 
@@ -98,7 +98,7 @@ Please :star: star :star: this project and help it grow! https://github.com/dgtl
 With Docker composer, just clone this repository and..
 
 ```bash
-$ docker-compose up -d
+$ docker compose up -d
 ```
 
 Docker standalone
@@ -137,10 +137,10 @@ docker rm $(docker ps -a -f name=changedetection.io -q)
 docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
 ```
 
-### docker-compose
+### docker compose
 
 ```bash
-docker-compose pull && docker-compose up -d
+docker compose pull && docker compose up -d
 ```
 
 See the wiki for more information https://github.com/dgtlmoon/changedetection.io/wiki

app.json

@@ -1,21 +0,0 @@
-{
-  "name": "ChangeDetection.io",
-  "description": "The best and simplest self-hosted open source website change detection monitoring and notification service.",
-  "keywords": [
-    "changedetection",
-    "website monitoring"
-  ],
-  "repository": "https://github.com/dgtlmoon/changedetection.io",
-  "success_url": "/",
-  "scripts": {
-  },
-  "env": {
-  },
-  "formation": {
-    "web": {
-      "quantity": 1,
-      "size": "free"
-    }
-  },
-  "image": "heroku/python"
-}

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.45.8.1'
+__version__ = '0.45.12'
 
 from distutils.util import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/api/api_v1.py

@@ -30,7 +30,7 @@ class Watch(Resource):
         self.update_q = kwargs['update_q']
 
     # Get information about a single watch, excluding the history list (can be large)
-    # curl http://localhost:4000/api/v1/watch/<string:uuid>
+    # curl http://localhost:5000/api/v1/watch/<string:uuid>
     # @todo - version2 - ?muted and ?paused should be able to be called together, return the watch struct not "OK"
     # ?recheck=true
     @auth.check_token
@@ -39,9 +39,9 @@ class Watch(Resource):
         @api {get} /api/v1/watch/:uuid Single watch - get data, recheck, pause, mute.
         @apiDescription Retrieve watch information and set muted/paused status
         @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            curl "http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?muted=unmuted"  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            curl "http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?paused=unpaused"  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl "http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?muted=unmuted"  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl "http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?paused=unpaused"  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
         @apiName Watch
         @apiGroup Watch
         @apiParam {uuid} uuid Watch unique ID.
@@ -84,7 +84,7 @@ class Watch(Resource):
         """
         @api {delete} /api/v1/watch/:uuid Delete a watch and related history
         @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45"
         @apiParam {uuid} uuid Watch unique ID.
         @apiName Delete
         @apiGroup Watch
@@ -103,7 +103,7 @@ class Watch(Resource):
         @api {put} /api/v1/watch/:uuid Update watch information
         @apiExample {curl} Example usage:
             Update (PUT)
-            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}'
+            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}'
 
         @apiDescription Updates an existing watch using JSON, accepts the same structure as returned in <a href="#api-Watch-Watch">get single watch information</a>
         @apiParam {uuid} uuid Watch unique ID.
@@ -132,13 +132,13 @@ class WatchHistory(Resource):
         self.datastore = kwargs['datastore']
 
     # Get a list of available history for a watch by UUID
-    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history
+    # curl http://localhost:5000/api/v1/watch/<string:uuid>/history
     def get(self, uuid):
         """
         @api {get} /api/v1/watch/<string:uuid>/history Get a list of all historical snapshots available for a watch
         @apiDescription Requires `uuid`, returns list
         @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
+            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
             {
                 "1676649279": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/cb7e9be8258368262246910e6a2a4c30.txt",
                 "1677092785": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/e20db368d6fc633e34f559ff67bb4044.txt",
@@ -166,7 +166,7 @@ class WatchSingleHistory(Resource):
         @api {get} /api/v1/watch/<string:uuid>/history/<int:timestamp> Get single snapshot from watch
         @apiDescription Requires watch `uuid` and `timestamp`. `timestamp` of "`latest`" for latest available snapshot, or <a href="#api-Watch_History-Get_list_of_available_stored_snapshots_for_watch">use the list returned here</a>
         @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
+            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
         @apiName Get single snapshot content
         @apiGroup Watch History
         @apiSuccess (200) {String} OK
@@ -202,7 +202,7 @@ class CreateWatch(Resource):
         @api {post} /api/v1/watch Create a single watch
         @apiDescription Requires atleast `url` set, can accept the same structure as <a href="#api-Watch-Watch">get single watch information</a> to create.
         @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
+            curl http://localhost:5000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
         @apiName Create
         @apiGroup Watch
         @apiSuccess (200) {String} OK Was created
@@ -245,7 +245,7 @@ class CreateWatch(Resource):
         @api {get} /api/v1/watch List watches
         @apiDescription Return concise list of available watches and some very basic info
         @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl http://localhost:5000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45"
             {
                 "6a4b7d5c-fee4-4616-9f43-4ac97046b595": {
                     "last_changed": 1677103794,
@@ -363,7 +363,7 @@ class SystemInfo(Resource):
         @api {get} /api/v1/systeminfo Return system info
         @apiDescription Return some info about the current system state
         @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl http://localhost:5000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45"
             HTTP/1.0 200
             {
                 'queue_size': 10 ,

changedetectionio/blueprint/browser_steps/__init__.py

@@ -58,7 +58,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             io_interface_context = io_interface_context.start()
 
         keepalive_ms = ((keepalive_seconds + 3) * 1000)
-        base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '')
+        base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '').strip('"')
         a = "?" if not '?' in base_url else '&'
         base_url += a + f"timeout={keepalive_ms}"
 

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -3,7 +3,7 @@
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>

changedetectionio/content_fetcher.py

@@ -43,9 +43,11 @@ class JSActionExceptions(Exception):
         return
 
 
-class BrowserStepsStepTimout(Exception):
-    def __init__(self, step_n):
+class BrowserStepsStepException(Exception):
+    def __init__(self, step_n, original_e):
         self.step_n = step_n
+        self.original_e = original_e
+        print(f"Browser Steps exception at step {self.step_n}", str(original_e))
         return
 
 
@@ -91,19 +93,20 @@ class ReplyWithContentButNoText(Exception):
 
 
 class Fetcher():
+    browser_connection_is_custom = None
+    browser_connection_url = None
     browser_steps = None
     browser_steps_screenshot_path = None
     content = None
     error = None
     fetcher_description = "No description"
-    browser_connection_url = None
     headers = {}
+    instock_data = None
+    instock_data_js = ""
     status_code = None
     webdriver_js_execute_code = None
     xpath_data = None
     xpath_element_js = ""
-    instock_data = None
-    instock_data_js = ""
 
     # Will be needed in the future by the VisualSelector, always get this where possible.
     screenshot = False
@@ -172,7 +175,7 @@ class Fetcher():
 
     def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-        from playwright._impl._errors import TimeoutError
+        from playwright._impl._errors import TimeoutError, Error
         from jinja2 import Environment
         jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
 
@@ -202,10 +205,10 @@ class Fetcher():
                                                       optional_value=optional_value)
                     self.screenshot_step(step_n)
                     self.save_step_html(step_n)
-                except TimeoutError as e:
-                    print(str(e))
+
+                except (Error, TimeoutError) as e:
                     # Stop processing here
-                    raise BrowserStepsStepTimout(step_n=step_n)
+                    raise BrowserStepsStepException(step_n=step_n, original_e=e)
 
     # It's always good to reset these
     def delete_browser_steps_screenshots(self):
@@ -252,16 +255,19 @@ class base_html_playwright(Fetcher):
 
     proxy = None
 
-    def __init__(self, proxy_override=None, browser_connection_url=None):
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
         super().__init__()
 
         self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
 
-        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-        if not browser_connection_url:
-            self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
+        if custom_browser_connection_url:
+            self.browser_connection_is_custom = True
+            self.browser_connection_url = custom_browser_connection_url
         else:
-            self.browser_connection_url = browser_connection_url
+            # Fallback to fetching from system
+            # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+            self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
+
 
         # If any proxy settings are enabled, then we should setup the proxy object
         proxy_args = {}
@@ -311,7 +317,6 @@ class base_html_playwright(Fetcher):
             is_binary=False):
 
         from pkg_resources import resource_string
-        from urllib.parse import urlparse
 
         extra_wait_ms = (int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay) * 1000
 
@@ -324,19 +329,13 @@ class base_html_playwright(Fetcher):
         from requests.exceptions import ConnectTimeout, ReadTimeout
         wait_browserless_seconds = 240
 
-        # Could be set from the browser-override
-        browserless_function_url = self.browser_connection_url
+        browserless_function_url = os.getenv('BROWSERLESS_FUNCTION_URL')
+        from urllib.parse import urlparse
+        if not browserless_function_url:
+            # Convert/try to guess from PLAYWRIGHT_DRIVER_URL
+            o = urlparse(os.getenv('PLAYWRIGHT_DRIVER_URL'))
+            browserless_function_url = o._replace(scheme="http")._replace(path="function").geturl()
 
-        # Or use the system config defaults
-        if not self.browser_connection_url:
-            browserless_function_url = os.getenv('BROWSERLESS_FUNCTION_URL')
-
-        if not self.browser_connection_url:
-            browserless_function_url = os.getenv('PLAYWRIGHT_DRIVER_URL')
-
-        # Convert always to http:// and function/ for browserless
-        o = urlparse(browserless_function_url)
-        browserless_function_url = o._replace(scheme="http")._replace(path="function").geturl()
 
         # Append proxy connect string
         if self.proxy:
@@ -428,8 +427,10 @@ class base_html_playwright(Fetcher):
             current_include_filters=None,
             is_binary=False):
 
+
         # For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!)
-        if not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
+        # browser_connection_is_custom doesnt work with puppeteer style fetch (use playwright native too in this case)
+        if not self.browser_connection_is_custom and not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
             if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')):
                 # Temporary backup solution until we rewrite the playwright code
                 return self.run_fetch_browserless_puppeteer(
@@ -576,15 +577,16 @@ class base_html_webdriver(Fetcher):
                                         'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
     proxy = None
 
-    def __init__(self, proxy_override=None, browser_connection_url=None):
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
         super().__init__()
         from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
 
         # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-        if not browser_connection_url:
+        if not custom_browser_connection_url:
             self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
         else:
-            self.browser_connection_url = browser_connection_url
+            self.browser_connection_is_custom = True
+            self.browser_connection_url = custom_browser_connection_url
 
         # If any proxy settings are enabled, then we should setup the proxy object
         proxy_args = {}
@@ -681,7 +683,7 @@ class base_html_webdriver(Fetcher):
 class html_requests(Fetcher):
     fetcher_description = "Basic fast Plaintext/HTTP Client"
 
-    def __init__(self, proxy_override=None, browser_connection_url=None):
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
         super().__init__()
         self.proxy_override = proxy_override
         # browser_connection_url is none because its always 'launched locally'

changedetectionio/flask_app.py

@@ -485,14 +485,18 @@ def changedetection_app(config=None, datastore_o=None):
 
 
     # AJAX endpoint for sending a test
+    @app.route("/notification/send-test/<string:watch_uuid>", methods=['POST'])
     @app.route("/notification/send-test", methods=['POST'])
+    @app.route("/notification/send-test/", methods=['POST'])
     @login_optionally_required
-    def ajax_callback_send_notification_test():
+    def ajax_callback_send_notification_test(watch_uuid=None):
 
+        # Watch_uuid could be unsuet in the case its used in tag editor, global setings
         import apprise
         from .apprise_asset import asset
         apobj = apprise.Apprise(asset=asset)
 
+        watch = datastore.data['watching'].get(watch_uuid) if watch_uuid else None
 
         # validate URLS
         if not len(request.form['notification_urls'].strip()):
@@ -505,9 +509,11 @@ def changedetection_app(config=None, datastore_o=None):
                     return make_response({'error': message}, 400)
 
         try:
-            n_object = {'watch_url': request.form['window_url'],
-                        'notification_urls': request.form['notification_urls'].splitlines()
-                        }
+            # use the same as when it is triggered, but then override it with the form test values
+            n_object = {
+                'watch_url': request.form['window_url'],
+                'notification_urls': request.form['notification_urls'].splitlines()
+            }
 
             # Only use if present, if not set in n_object it should use the default system value
             if 'notification_format' in request.form and request.form['notification_format'].strip():
@@ -519,7 +525,9 @@ def changedetection_app(config=None, datastore_o=None):
             if 'notification_body' in request.form and request.form['notification_body'].strip():
                 n_object['notification_body'] = request.form.get('notification_body', '').strip()
 
-            notification_q.put(n_object)
+            from . import update_worker
+            new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
+            new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch)
         except Exception as e:
             return make_response({'error': str(e)}, 400)
 
@@ -1584,6 +1592,15 @@ def notification_runner():
 
             try:
                 from changedetectionio import notification
+                # Fallback to system config if not set
+                if not n_object.get('notification_body') and datastore.data['settings']['application'].get('notification_body'):
+                    n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
+
+                if not n_object.get('notification_title') and datastore.data['settings']['application'].get('notification_title'):
+                    n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
+
+                if not n_object.get('notification_format') and datastore.data['settings']['application'].get('notification_format'):
+                    n_object['notification_title'] = datastore.data['settings']['application'].get('notification_format')
 
                 sent_obj = notification.process_notification(n_object, datastore)
 

changedetectionio/forms.py

@@ -43,6 +43,7 @@ valid_method = {
     'PUT',
     'PATCH',
     'DELETE',
+    'OPTIONS',
 }
 
 default_method = 'GET'

changedetectionio/model/Watch.py

@@ -38,6 +38,7 @@ base_config = {
     'track_ldjson_price_data': None,
     'headers': {},  # Extra headers to send
     'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
+    'in_stock' : None,
     'in_stock_only' : True, # Only trigger change on going to instock from out-of-stock
     'include_filters': [],
     'last_checked': 0,

changedetectionio/notification.py

@@ -1,4 +1,5 @@
 import apprise
+import time
 from jinja2 import Environment, BaseLoader
 from apprise import NotifyFormat
 import json
@@ -119,8 +120,8 @@ def process_notification(n_object, datastore):
 
     # Get the notification body from datastore
     jinja2_env = Environment(loader=BaseLoader)
-    n_body = jinja2_env.from_string(n_object.get('notification_body', default_notification_body)).render(**notification_parameters)
-    n_title = jinja2_env.from_string(n_object.get('notification_title', default_notification_title)).render(**notification_parameters)
+    n_body = jinja2_env.from_string(n_object.get('notification_body', '')).render(**notification_parameters)
+    n_title = jinja2_env.from_string(n_object.get('notification_title', '')).render(**notification_parameters)
     n_format = valid_notification_formats.get(
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
@@ -131,103 +132,108 @@ def process_notification(n_object, datastore):
         # Initially text or whatever
         n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
 
-
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
     # Anything higher than or equal to WARNING (which covers things like Connection errors)
     # raise it as an exception
-    apobjs=[]
-    sent_objs=[]
+
+    sent_objs = []
     from .apprise_asset import asset
-    for url in n_object['notification_urls']:
-        url = jinja2_env.from_string(url).render(**notification_parameters)
-        apobj = apprise.Apprise(debug=True, asset=asset)
-        url = url.strip()
-        if len(url):
+    apobj = apprise.Apprise(debug=True, asset=asset)
+
+    if not n_object.get('notification_urls'):
+        return None
+
+    with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
+        for url in n_object['notification_urls']:
+            url = url.strip()
             print(">> Process Notification: AppRise notifying {}".format(url))
-            with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
-                # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
-                # Because different notifications may require different pre-processing, run each sequentially :(
-                # 2000 bytes minus -
-                #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
-                #     Length of URL - Incase they specify a longer custom avatar_url
+            url = jinja2_env.from_string(url).render(**notification_parameters)
 
-                # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
-                k = '?' if not '?' in url else '&'
-                if not 'avatar_url' in url \
-                        and not url.startswith('mail') \
-                        and not url.startswith('post') \
-                        and not url.startswith('get') \
-                        and not url.startswith('delete') \
-                        and not url.startswith('put'):
-                    url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
+            # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
+            # Because different notifications may require different pre-processing, run each sequentially :(
+            # 2000 bytes minus -
+            #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
+            #     Length of URL - Incase they specify a longer custom avatar_url
 
-                if url.startswith('tgram://'):
-                    # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
-                    # re https://github.com/dgtlmoon/changedetection.io/issues/555
-                    # @todo re-use an existing library we have already imported to strip all non-allowed tags
-                    n_body = n_body.replace('<br>', '\n')
-                    n_body = n_body.replace('</br>', '\n')
-                    # real limit is 4096, but minus some for extra metadata
-                    payload_max_size = 3600
-                    body_limit = max(0, payload_max_size - len(n_title))
-                    n_title = n_title[0:payload_max_size]
-                    n_body = n_body[0:body_limit]
+            # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
+            k = '?' if not '?' in url else '&'
+            if not 'avatar_url' in url \
+                    and not url.startswith('mail') \
+                    and not url.startswith('post') \
+                    and not url.startswith('get') \
+                    and not url.startswith('delete') \
+                    and not url.startswith('put'):
+                url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
 
-                elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith('https://discord.com/api'):
-                    # real limit is 2000, but minus some for extra metadata
-                    payload_max_size = 1700
-                    body_limit = max(0, payload_max_size - len(n_title))
-                    n_title = n_title[0:payload_max_size]
-                    n_body = n_body[0:body_limit]
+            if url.startswith('tgram://'):
+                # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
+                # re https://github.com/dgtlmoon/changedetection.io/issues/555
+                # @todo re-use an existing library we have already imported to strip all non-allowed tags
+                n_body = n_body.replace('<br>', '\n')
+                n_body = n_body.replace('</br>', '\n')
+                # real limit is 4096, but minus some for extra metadata
+                payload_max_size = 3600
+                body_limit = max(0, payload_max_size - len(n_title))
+                n_title = n_title[0:payload_max_size]
+                n_body = n_body[0:body_limit]
 
-                elif url.startswith('mailto'):
-                    # Apprise will default to HTML, so we need to override it
-                    # So that whats' generated in n_body is in line with what is going to be sent.
-                    # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
-                    if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
-                        prefix = '?' if not '?' in url else '&'
-                        # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
-                        n_format = n_format.tolower()
-                        url = "{}{}format={}".format(url, prefix, n_format)
-                    # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
+            elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith(
+                    'https://discord.com/api'):
+                # real limit is 2000, but minus some for extra metadata
+                payload_max_size = 1700
+                body_limit = max(0, payload_max_size - len(n_title))
+                n_title = n_title[0:payload_max_size]
+                n_body = n_body[0:body_limit]
 
-                apobj.add(url)
+            elif url.startswith('mailto'):
+                # Apprise will default to HTML, so we need to override it
+                # So that whats' generated in n_body is in line with what is going to be sent.
+                # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
+                if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
+                    prefix = '?' if not '?' in url else '&'
+                    # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
+                    n_format = n_format.lower()
+                    url = f"{url}{prefix}format={n_format}"
+                # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
 
-                apobj.notify(
-                    title=n_title,
-                    body=n_body,
-                    body_format=n_format,
-                    # False is not an option for AppRise, must be type None
-                    attach=n_object.get('screenshot', None)
-                )
+            apobj.add(url)
 
-                apobj.clear()
+            sent_objs.append({'title': n_title,
+                              'body': n_body,
+                              'url': url,
+                              'body_format': n_format})
 
-                # Incase it needs to exist in memory for a while after to process(?)
-                apobjs.append(apobj)
+        # Blast off the notifications tht are set in .add()
+        apobj.notify(
+            title=n_title,
+            body=n_body,
+            body_format=n_format,
+            # False is not an option for AppRise, must be type None
+            attach=n_object.get('screenshot', None)
+        )
 
-                # Returns empty string if nothing found, multi-line string otherwise
-                log_value = logs.getvalue()
-                if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
-                    raise Exception(log_value)
+        # Give apprise time to register an error
+        time.sleep(3)
 
-                sent_objs.append({'title': n_title,
-                                  'body': n_body,
-                                  'url' : url,
-                                  'body_format': n_format})
+        # Returns empty string if nothing found, multi-line string otherwise
+        log_value = logs.getvalue()
+
+        if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
+            raise Exception(log_value)
 
     # Return what was sent for better logging - after the for loop
     return sent_objs
 
 
 # Notification title + body content parameters get created here.
+# ( Where we prepare the tokens in the notification to be replaced with actual values )
 def create_notification_parameters(n_object, datastore):
     from copy import deepcopy
 
     # in the case we send a test notification from the main settings, there is no UUID.
     uuid = n_object['uuid'] if 'uuid' in n_object else ''
 
-    if uuid != '':
+    if uuid:
         watch_title = datastore.data['watching'][uuid].get('title', '')
         tag_list = []
         tags = datastore.get_all_tags_for_watch(uuid)
@@ -255,7 +261,7 @@ def create_notification_parameters(n_object, datastore):
     tokens.update(
         {
             'base_url': base_url,
-            'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '',
+            'current_snapshot': n_object.get('current_snapshot', ''),
             'diff': n_object.get('diff', ''),  # Null default in the case we use a test
             'diff_added': n_object.get('diff_added', ''),  # Null default in the case we use a test
             'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test

changedetectionio/processors/__init__.py

@@ -43,14 +43,14 @@ class difference_detection_processor():
 
         # In the case that the preferred fetcher was a browser config with custom connection URL..
         # @todo - on save watch, if its extra_browser_ then it should be obvious it will use playwright (like if its requests now..)
-        browser_connection_url = None
+        custom_browser_connection_url = None
         if prefer_fetch_backend.startswith('extra_browser_'):
             (t, key) = prefer_fetch_backend.split('extra_browser_')
             connection = list(
                 filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', [])))
             if connection:
                 prefer_fetch_backend = 'base_html_playwright'
-                browser_connection_url = connection[0].get('browser_connection_url')
+                custom_browser_connection_url = connection[0].get('browser_connection_url')
 
         # PDF should be html_requests because playwright will serve it up (so far) in a embedded page
         # @todo https://github.com/dgtlmoon/changedetection.io/issues/2019
@@ -74,7 +74,7 @@ class difference_detection_processor():
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
         self.fetcher = fetcher_obj(proxy_override=proxy_url,
-                                   browser_connection_url=browser_connection_url
+                                   custom_browser_connection_url=custom_browser_connection_url
                                    )
 
         if self.watch.has_browser_steps:

changedetectionio/res/stock-not-in-stock.js

@@ -4,6 +4,7 @@ function isItemInStock() {
     ' أخبرني عندما يتوفر',
     '0 in stock',
     'agotado',
+    'article épuisé',
     'artikel zurzeit vergriffen',
     'as soon as stock is available',
     'ausverkauft', // sold out
@@ -17,10 +18,8 @@ function isItemInStock() {
     'currently have any tickets for this',
     'currently unavailable',
     'dostępne wkrótce',
-    'dostępne wkrótce',
     'en rupture de stock',
     'ist derzeit nicht auf lager',
-    'ist derzeit nicht auf lager',
     'item is no longer available',
     'let me know when it\'s available',
     'message if back in stock',
@@ -28,6 +27,8 @@ function isItemInStock() {
     'nicht auf lager',
     'nicht lieferbar',
     'nicht zur verfügung',
+    'niet beschikbaar',
+    'niet leverbaar',
     'no disponible temporalmente',
     'no longer in stock',
     'no tickets available',
@@ -44,6 +45,7 @@ function isItemInStock() {
     'temporarily out of stock',
     'temporarily unavailable',
     'tickets unavailable',
+    'tijdelijk uitverkocht',
     'unavailable tickets',
     'we do not currently have an estimate of when this product will be back in stock.',
     'zur zeit nicht an lager',

changedetectionio/static/js/notifications.js

@@ -24,14 +24,17 @@ $(document).ready(function() {
     })
 
     data = {
-        window_url : window.location.href,
-        notification_urls : $('.notification-urls').val(),
+      notification_body: $('#notification_body').val(),
+      notification_format: $('#notification_format').val(),
+      notification_title: $('#notification_title').val(),
+      notification_urls: $('.notification-urls').val(),
+      window_url: window.location.href,
     }
-    for (key in data) {
-      if (!data[key].length) {
-        alert(key+" is empty, cannot send test.")
-        return;
-      }
+
+
+    if (!data['notification_urls'].length) {
+      alert("Notification URL list is empty, cannot send test.")
+      return;
     }
 
     $.ajax({

changedetectionio/store.py

@@ -248,6 +248,7 @@ class ChangeDetectionStore:
                 'check_count': 0,
                 'fetch_time' : 0.0,
                 'has_ldjson_price_data': None,
+                'in_stock': None,
                 'last_checked': 0,
                 'last_error': False,
                 'last_notification_error': False,

changedetectionio/templates/edit.html

@@ -14,7 +14,7 @@
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
 {% endif %}
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
     const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
     const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
     const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";

changedetectionio/templates/settings.html

@@ -4,7 +4,7 @@
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
 {% endif %}

changedetectionio/templates/watch-overview.html

@@ -141,7 +141,7 @@
                     {% if watch['processor'] == 'restock_diff'  %}
                     <span class="restock-label {{'in-stock' if watch['in_stock'] else 'not-in-stock' }}" title="detecting restock conditions">
                         <!-- maybe some object watch['processor'][restock_diff] or.. -->
-                        {% if watch['last_checked'] %}
+                        {% if watch['last_checked'] and watch['in_stock'] != None %}
                             {% if watch['in_stock'] %} In stock {% else %} Not in stock {% endif %}
                         {% else %}
                             Not yet checked

changedetectionio/tests/test_notification_errors.py

@@ -1,8 +1,7 @@
 import os
 import time
-import re
 from flask import url_for
-from . util import set_original_response, set_modified_response, live_server_setup
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 import logging
 
 def test_check_notification_error_handling(client, live_server):
@@ -11,7 +10,7 @@ def test_check_notification_error_handling(client, live_server):
     set_original_response()
 
     # Give the endpoint time to spin up
-    time.sleep(2)
+    time.sleep(1)
 
     # Set a URL and fetch it, then set a notification URL which is going to give errors
     test_url = url_for('test_endpoint', _external=True)
@@ -22,12 +21,16 @@ def test_check_notification_error_handling(client, live_server):
     )
     assert b"Watch added" in res.data
 
-    time.sleep(2)
+    wait_for_all_checks(client)
     set_modified_response()
 
+    working_notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
+    broken_notification_url = "jsons://broken-url-xxxxxxxx123/test"
+
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"notification_urls": "jsons://broken-url-xxxxxxxx123/test",
+        # A URL with errors should not block the one that is working
+        data={"notification_urls": f"{broken_notification_url}\r\n{working_notification_url}",
               "notification_title": "xxx",
               "notification_body": "xxxxx",
               "notification_format": "Text",
@@ -63,4 +66,10 @@ def test_check_notification_error_handling(client, live_server):
     found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
     assert found_name_resolution_error
 
+    # And the working one, which is after the 'broken' one should still have fired
+    with open("test-datastore/notification.txt", "r") as f:
+        notification_submission = f.read()
+    os.unlink("test-datastore/notification.txt")
+    assert 'xxxxx' in notification_submission
+
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/update_worker.py

@@ -26,47 +26,61 @@ class update_worker(threading.Thread):
         self.datastore = datastore
         super().__init__(*args, **kwargs)
 
-    def queue_notification_for_watch(self, n_object, watch):
+    def queue_notification_for_watch(self, notification_q, n_object, watch):
 
         from changedetectionio import diff
+        dates = []
+        trigger_text = ''
+
+        if watch:
+            watch_history = watch.history
+            dates = list(watch_history.keys())
+            trigger_text = watch.get('trigger_text', [])
 
-        watch_history = watch.history
-        dates = list(watch_history.keys())
         # Add text that was triggered
-        snapshot_contents = watch.get_history_snapshot(dates[-1])
+        if len(dates):
+            snapshot_contents = watch.get_history_snapshot(dates[-1])
+        else:
+            snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
 
         # HTML needs linebreak, but MarkDown and Text can use a linefeed
-        if n_object['notification_format'] == 'HTML':
+        if n_object.get('notification_format') == 'HTML':
             line_feed_sep = "<br>"
             # Snapshot will be plaintext on the disk, convert to some kind of HTML
             snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
         else:
             line_feed_sep = "\n"
 
-        trigger_text = watch.get('trigger_text', [])
         triggered_text = ''
-
         if len(trigger_text):
             from . import html_tools
             triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
             if triggered_text:
                 triggered_text = line_feed_sep.join(triggered_text)
 
+        # Could be called as a 'test notification' with only 1 snapshot available
+        prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
+        current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
+
+        if len(dates) > 1:
+            prev_snapshot = watch.get_history_snapshot(dates[-2])
+            current_snapshot = watch.get_history_snapshot(dates[-1])
 
         n_object.update({
             'current_snapshot': snapshot_contents,
-            'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
-            'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
-            'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
-            'diff_patch': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep, patch_format=True),
-            'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
-            'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
+            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep),
+            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
+            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep),
+            'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
+            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
+            'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
             'triggered_text': triggered_text,
-            'uuid': watch.get('uuid'),
-            'watch_url': watch.get('url'),
+            'uuid': watch.get('uuid') if watch else None,
+            'watch_url': watch.get('url') if watch else None,
         })
-        logging.info (">> SENDING NOTIFICATION")
-        self.notification_q.put(n_object)
+        logging.info(">> SENDING NOTIFICATION")
+
+        notification_q.put(n_object)
 
     # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
     def _check_cascading_vars(self, var_name, watch):
@@ -134,7 +148,7 @@ class update_worker(threading.Thread):
         queued = False
         if n_object and n_object.get('notification_urls'):
             queued = True
-            self.queue_notification_for_watch(n_object, watch)
+            self.queue_notification_for_watch(notification_q=self.notification_q, n_object=n_object, watch=watch)
 
         return queued
 
@@ -174,9 +188,9 @@ class update_worker(threading.Thread):
             return
         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
         n_object = {'notification_title': "Changedetection.io - Alert - Browser step at position {} could not be run".format(step_n+1),
-                    'notification_body': "Your configured browser step at position {} for {{watch['url']}} "
+                    'notification_body': "Your configured browser step at position {} for {{{{watch_url}}}} "
                                          "did not appear on the page after {} attempts, did the page change layout? "
-                                         "Does it need a delay added?\n\nLink: {{base_url}}/edit/{{watch_uuid}}\n\n"
+                                         "Does it need a delay added?\n\nLink: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\n"
                                          "Thanks - Your omniscient changedetection.io installation :)\n".format(step_n+1, threshold),
                     'notification_format': 'text'}
 
@@ -340,20 +354,32 @@ class update_worker(threading.Thread):
                         changed_detected = False
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False})
 
-                    except content_fetcher.BrowserStepsStepTimout as e:
+                    except content_fetcher.BrowserStepsStepException as e:
 
                         if not self.datastore.data['watching'].get(uuid):
                             continue
 
                         error_step = e.step_n + 1
-                        err_text = f"Warning, browser step at position {error_step} could not run, target not found, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step"
+                        from playwright._impl._errors import TimeoutError, Error
+
+                        # Generally enough info for TimeoutError (couldnt locate the element after default seconds)
+                        err_text = f"Browser step at position {error_step} could not run, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step."
+
+                        if e.original_e.name == "TimeoutError":
+                            # Just the first line is enough, the rest is the stack trace
+                            err_text += " Could not find the target."
+                        else:
+                            # Other Error, more info is good.
+                            err_text += " " + str(e.original_e).splitlines()[0]
+
+                        print(f"BrowserSteps exception at step {error_step}", str(e.original_e))
+
                         self.datastore.update_watch(uuid=uuid,
                                                     update_obj={'last_error': err_text,
                                                                 'browser_steps_last_error_step': error_step
                                                                 }
                                                     )
 
-
                         if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
                             c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
                             c += 1

docker-compose.yml

@@ -90,7 +90,7 @@ services:
 #
 
      # Used for fetching pages via Playwright+Chrome where you need Javascript support.
-     # Note: works well but is deprecated, doesnt fetch full page screenshots and other issues
+     # Note: works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector) and other issues
 #    browser-chrome:
 #        hostname: browser-chrome
 #        image: selenium/standalone-chrome:4

heroku.yml

@@ -1,2 +0,0 @@
-run:
-  changedetection: python3 ./changedetection.py -C -d ./datastore -p $PORT

requirements.txt

@@ -30,7 +30,7 @@ dnspython~=2.4 # related to eventlet fixes
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise~=1.6.0
+apprise~=1.7.1
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt

setup.py

@@ -27,7 +27,7 @@ install_requires = open('requirements.txt').readlines()
 setup(
     name='changedetection.io',
     version=find_version("changedetectionio", "__init__.py"),
-    description='Website change detection and monitoring service',
+    description='Website change detection and monitoring service, detect changes to web pages and send alerts/notifications.',
     long_description=open('README-pip.md').read(),
     long_description_content_type='text/markdown',
     keywords='website change monitor for changes notification change detection '
@@ -41,7 +41,7 @@ setup(
     include_package_data=True,
     install_requires=install_requires,
     license="Apache License 2.0",
-    python_requires=">= 3.7",
+    python_requires=">= 3.10",
     classifiers=['Intended Audience :: Customer Service',
                  'Intended Audience :: Developers',
                  'Intended Audience :: Education',