0.45.11

.github/workflows/pypi-release.yml

@@ -0,0 +1,72 @@
+name: Publish Python 🐍distribution 📦 to PyPI and TestPyPI
+
+on: push
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
+    - name: Install pypa/build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: python3 -m build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+
+
+  test-pypi-package:
+    name: Test the built 📦 package works basically.
+    runs-on: ubuntu-latest
+    needs:
+    - build
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Test that the basic pip built package runs without error
+      run: |
+        set -e
+        pip3 install dist/changedetection.io*.whl
+        changedetection.io -d /tmp -p 10000 &
+        sleep 3
+        curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
+        curl http://127.0.0.1:10000/ >/dev/null
+        killall changedetection.io
+
+
+  publish-to-pypi:
+    name: >-
+      Publish Python 🐍 distribution 📦 to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - test-pypi-package
+    runs-on: ubuntu-latest
+    environment:
+      name: release
+      url: https://pypi.org/p/changedetection.io
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/test-only.yml

@@ -98,14 +98,58 @@ jobs:
 
       - name: Test changedetection.io container starts+runs basically without error
         run: |
-          docker run -p 5556:5000 -d test-changedetectionio
+          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
           sleep 3
           # Should return 0 (no error) when grep finds it
           curl -s http://localhost:5556 |grep -q checkbox-uuid
           
           # and IPv6
           curl -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+          docker kill test-changedetectionio
+
+      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
+        run: |
           
+          echo SIGINT Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGINT to sig-test container"
+          docker kill --signal=SIGINT sig-test
+          sleep 3
+          # invert the check (it should be not 0/not running)
+          docker ps
+          # check signal catch(STDOUT) log
+          docker logs sig-test | grep 'Shutdown: Got Signal - SIGINT' || exit 1
+          test -z "`docker ps|grep sig-test`"
+          if [ $? -ne 0 ]
+          then
+            echo "Looks like container was running when it shouldnt be"
+            docker ps
+            exit 1
+          fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists 
+          docker rm sig-test
+          
+          echo SIGTERM Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGTERM to sig-test container"
+          docker kill --signal=SIGTERM sig-test
+          sleep 3
+          # invert the check (it should be not 0/not running)
+          docker ps
+          docker logs sig-test | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
+          test -z "`docker ps|grep sig-test`"
+          if [ $? -ne 0 ]
+          then
+            echo "Looks like container was running when it shouldnt be"
+            docker ps
+            exit 1
+          fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists           
+          docker rm sig-test
 
 #export WEBDRIVER_URL=http://localhost:4444/wd/hub
 #pytest tests/fetchers/test_content.py

.github/workflows/test-pip-build.yml

@@ -1,36 +0,0 @@
-name: ChangeDetection.io PIP package test
-
-# Triggers the workflow on push or pull request events
-
-# This line doesnt work, even tho it is the documented one
-on: [push, pull_request]
-
-  # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
-  # @todo: some kind of path filter for requirements.txt and Dockerfile
-jobs:
-  test-pip-build-basics:
-    runs-on: ubuntu-latest
-    steps:
-        - uses: actions/checkout@v4
-
-        - name: Set up Python 3.11
-          uses: actions/setup-python@v4
-          with:
-            python-version: 3.11
-
-
-        - name: Test that the basic pip built package runs without error
-          run: |
-            set -e
-            mkdir dist
-            pip3 install wheel
-            python3 setup.py bdist_wheel            
-            pip3 install -r requirements.txt
-            rm ./changedetection.py
-            rm -rf changedetectio
-            
-            pip3 install dist/changedetection.io*.whl
-            changedetection.io -d /tmp -p 10000 &
-            sleep 3
-            curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
-            killall -9 changedetection.io

Dockerfile

@@ -53,12 +53,12 @@ ENV PYTHONPATH=/usr/local
 
 EXPOSE 5000
 
-# The actual flask app
+# The actual flask app module
 COPY changedetectionio /app/changedetectionio
-
-# The eventlet server wrapper
+# Starting wrapper
 COPY changedetection.py /app/changedetection.py
 
 WORKDIR /app
+CMD ["python", "./changedetection.py", "-d", "/datastore"]
+
 
-CMD [ "python", "./changedetection.py" , "-d", "/datastore"]

MANIFEST.in

@@ -10,6 +10,8 @@ prune changedetectionio/static/package-lock.json
 prune changedetectionio/static/styles/node_modules
 prune changedetectionio/static/styles/package-lock.json
 include changedetection.py
+include requirements.txt
+include README-pip.md
 global-exclude *.pyc
 global-exclude node_modules
 global-exclude venv

Procfile

@@ -1 +0,0 @@
-web: python3 ./changedetection.py -C -d ./datastore -p $PORT

README.md

@@ -11,12 +11,13 @@ _Live your data-life pro-actively._
 
 ![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
 
-[**Don't have time? Let us host it for you! try our $8.99/month subscription - use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_
+[**Get started with website page change monitoring straight away. Don't have time? Try our $8.99/month subscription, use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_
 
 - Chrome browser included.
+- Nothing to install, access via browser login after signup.
 - Super fast, no registration needed setup.
 - Get started watching and receiving website change notifications straight away.
-
+- See our [tutorials and how-to page for more inspiration](https://changedetection.io/tutorials) 
 
 ### Target specific parts of the webpage using the Visual Selector tool.
 
@@ -97,7 +98,7 @@ Please :star: star :star: this project and help it grow! https://github.com/dgtl
 With Docker composer, just clone this repository and..
 
 ```bash
-$ docker-compose up -d
+$ docker compose up -d
 ```
 
 Docker standalone
@@ -136,10 +137,10 @@ docker rm $(docker ps -a -f name=changedetection.io -q)
 docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
 ```
 
-### docker-compose
+### docker compose
 
 ```bash
-docker-compose pull && docker-compose up -d
+docker compose pull && docker compose up -d
 ```
 
 See the wiki for more information https://github.com/dgtlmoon/changedetection.io/wiki

app.json

@@ -1,21 +0,0 @@
-{
-  "name": "ChangeDetection.io",
-  "description": "The best and simplest self-hosted open source website change detection monitoring and notification service.",
-  "keywords": [
-    "changedetection",
-    "website monitoring"
-  ],
-  "repository": "https://github.com/dgtlmoon/changedetection.io",
-  "success_url": "/",
-  "scripts": {
-  },
-  "env": {
-  },
-  "formation": {
-    "web": {
-      "quantity": 1,
-      "size": "free"
-    }
-  },
-  "image": "heroku/python"
-}

changedetection.py

@@ -1,44 +1,6 @@
 #!/usr/bin/python3
 
-# Entry-point for running from the CLI when not installed via Pip, Pip will handle the console_scripts entry_points's from setup.py
-# It's recommended to use `pip3 install changedetection.io` and start with `changedetection.py` instead, it will be linkd to your global path.
-# or Docker.
-# Read more https://github.com/dgtlmoon/changedetection.io/wiki
+# Only exists for direct CLI usage
 
-from changedetectionio import changedetection
-import multiprocessing
-import sys
-import os
-
-def sigchld_handler(_signo, _stack_frame):
-    import sys
-    print('Shutdown: Got SIGCHLD')
-    # https://stackoverflow.com/questions/40453496/python-multiprocessing-capturing-signals-to-restart-child-processes-or-shut-do
-    pid, status = os.waitpid(-1, os.WNOHANG | os.WUNTRACED | os.WCONTINUED)
-
-    print('Sub-process: pid %d status %d' % (pid, status))
-    if status != 0:
-        sys.exit(1)
-
-    raise SystemExit
-
-if __name__ == '__main__':
-
-    #signal.signal(signal.SIGCHLD, sigchld_handler)
-
-    # The only way I could find to get Flask to shutdown, is to wrap it and then rely on the subsystem issuing SIGTERM/SIGKILL
-    parse_process = multiprocessing.Process(target=changedetection.main)
-    parse_process.daemon = True
-    parse_process.start()
-    import time
-
-    try:
-        while True:
-            time.sleep(1)
-            if not parse_process.is_alive():
-                # Process died/crashed for some reason, exit with error set
-                sys.exit(1)
-
-    except KeyboardInterrupt:
-        #parse_process.terminate() not needed, because this process will issue it to the sub-process anyway
-        print ("Exited - CTRL+C")
+import changedetectionio
+changedetectionio.main()

changedetectionio/api/api_v1.py

@@ -296,6 +296,61 @@ class CreateWatch(Resource):
 
         return list, 200
 
+class Import(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    def post(self):
+        """
+        @api {post} /api/v1/import Import a list of watched URLs
+        @apiDescription Accepts a line-feed separated list of URLs to import, additionally with ?tag_uuids=(tag  id), ?tag=(name), ?proxy={key}, ?dedupe=true (default true) one URL per line.
+        @apiExample {curl} Example usage:
+            curl http://localhost:5000/api/v1/import --data-binary @list-of-sites.txt -H"x-api-key:8a111a21bc2f8f1dd9b9353bbd46049a"
+        @apiName Import
+        @apiGroup Watch
+        @apiSuccess (200) {List} OK List of watch UUIDs added
+        @apiSuccess (500) {String} ERR Some other error
+        """
+
+        extras = {}
+
+        if request.args.get('proxy'):
+            plist = self.datastore.proxy_list
+            if not request.args.get('proxy') in plist:
+                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
+            else:
+                extras['proxy'] = request.args.get('proxy')
+
+        dedupe = strtobool(request.args.get('dedupe', 'true'))
+
+        tags = request.args.get('tag')
+        tag_uuids = request.args.get('tag_uuids')
+
+        if tag_uuids:
+            tag_uuids = tag_uuids.split(',')
+
+        urls = request.get_data().decode('utf8').splitlines()
+        added = []
+        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
+        for url in urls:
+            url = url.strip()
+            if not len(url):
+                continue
+
+            # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+            if not validators.url(url, simple_host=allow_simplehost):
+                return f"Invalid or unsupported URL - {url}", 400
+
+            if dedupe and self.datastore.url_exists(url):
+                continue
+
+            new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags, tag_uuids=tag_uuids)
+            added.append(new_uuid)
+
+        return added
+
 class SystemInfo(Resource):
     def __init__(self, **kwargs):
         # datastore is a black box dependency

changedetectionio/blueprint/browser_steps/__init__.py

@@ -25,10 +25,9 @@ from distutils.util import strtobool
 from flask import Blueprint, request, make_response
 import logging
 import os
-import re
 
 from changedetectionio.store import ChangeDetectionStore
-from changedetectionio import login_optionally_required
+from changedetectionio.flask_app import login_optionally_required
 
 browsersteps_sessions = {}
 io_interface_context = None

changedetectionio/blueprint/tags/__init__.py

@@ -1,6 +1,6 @@
 from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
 from changedetectionio.store import ChangeDetectionStore
-from changedetectionio import login_optionally_required
+from changedetectionio.flask_app import login_optionally_required
 
 
 def construct_blueprint(datastore: ChangeDetectionStore):

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -3,7 +3,7 @@
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>

changedetectionio/changedetection.py

@@ -1,153 +0,0 @@
-#!/usr/bin/python3
-
-# Launch as a eventlet.wsgi server instance.
-
-from distutils.util import strtobool
-from json.decoder import JSONDecodeError
-
-import eventlet
-import eventlet.wsgi
-import getopt
-import os
-import signal
-import socket
-import sys
-
-from . import store, changedetection_app, content_fetcher
-from . import __version__
-
-# Only global so we can access it in the signal handler
-app = None
-datastore = None
-
-def sigterm_handler(_signo, _stack_frame):
-    global app
-    global datastore
-#    app.config.exit.set()
-    print('Shutdown: Got SIGTERM, DB saved to disk')
-    datastore.sync_to_json()
-#    raise SystemExit
-
-def main():
-    global datastore
-    global app
-
-    datastore_path = None
-    do_cleanup = False
-    host = ''
-    ipv6_enabled = False
-    port = os.environ.get('PORT') or 5000
-    ssl_mode = False
-
-    # On Windows, create and use a default path.
-    if os.name == 'nt':
-        datastore_path = os.path.expandvars(r'%APPDATA%\changedetection.io')
-        os.makedirs(datastore_path, exist_ok=True)
-    else:
-        # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
-        datastore_path = os.path.join(os.getcwd(), "../datastore")
-
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:", "port")
-    except getopt.GetoptError:
-        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
-        sys.exit(2)
-
-    create_datastore_dir = False
-
-    for opt, arg in opts:
-        if opt == '-s':
-            ssl_mode = True
-
-        if opt == '-h':
-            host = arg
-
-        if opt == '-p':
-            port = int(arg)
-
-        if opt == '-d':
-            datastore_path = arg
-
-        if opt == '-6':
-            print ("Enabling IPv6 listen support")
-            ipv6_enabled = True
-
-        # Cleanup (remove text files that arent in the index)
-        if opt == '-c':
-            do_cleanup = True
-
-        # Create the datadir if it doesnt exist
-        if opt == '-C':
-            create_datastore_dir = True
-
-    # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
-    app_config = {'datastore_path': datastore_path}
-
-    if not os.path.isdir(app_config['datastore_path']):
-        if create_datastore_dir:
-            os.mkdir(app_config['datastore_path'])
-        else:
-            print(
-                "ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists or specify a directory with the -d option.\n"
-                "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
-            sys.exit(2)
-
-    try:
-        datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
-    except JSONDecodeError as e:
-        # Dont' start if the JSON DB looks corrupt
-        print ("ERROR: JSON DB or Proxy List JSON at '{}' appears to be corrupt, aborting".format(app_config['datastore_path']))
-        print(str(e))
-        return
-
-    app = changedetection_app(app_config, datastore)
-
-    signal.signal(signal.SIGTERM, sigterm_handler)
-
-    # Go into cleanup mode
-    if do_cleanup:
-        datastore.remove_unused_snapshots()
-
-    app.config['datastore_path'] = datastore_path
-
-
-    @app.context_processor
-    def inject_version():
-        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
-                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
-                    has_password=datastore.data['settings']['application']['password'] != False
-                    )
-
-    # Monitored websites will not receive a Referer header when a user clicks on an outgoing link.
-    # @Note: Incompatible with password login (and maybe other features) for now, submit a PR!
-    @app.after_request
-    def hide_referrer(response):
-        if strtobool(os.getenv("HIDE_REFERER", 'false')):
-            response.headers["Referrer-Policy"] = "no-referrer"
-
-        return response
-
-    # Proxy sub-directory support
-    # Set environment var USE_X_SETTINGS=1 on this script
-    # And then in your proxy_pass settings
-    #
-    #         proxy_set_header Host "localhost";
-    #         proxy_set_header X-Forwarded-Prefix /app;
-
-    if os.getenv('USE_X_SETTINGS'):
-        print ("USE_X_SETTINGS is ENABLED\n")
-        from werkzeug.middleware.proxy_fix import ProxyFix
-        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
-
-    s_type = socket.AF_INET6 if ipv6_enabled else socket.AF_INET
-
-    if ssl_mode:
-        # @todo finalise SSL config, but this should get you in the right direction if you need it.
-        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port), s_type),
-                                               certfile='cert.pem',
-                                               keyfile='privkey.pem',
-                                               server_side=True), app)
-
-    else:
-        eventlet.wsgi.server(eventlet.listen((host, int(port)), s_type), app)
-

changedetectionio/content_fetcher.py

@@ -43,9 +43,11 @@ class JSActionExceptions(Exception):
         return
 
 
-class BrowserStepsStepTimout(Exception):
-    def __init__(self, step_n):
+class BrowserStepsStepException(Exception):
+    def __init__(self, step_n, original_e):
         self.step_n = step_n
+        self.original_e = original_e
+        print(f"Browser Steps exception at step {self.step_n}", str(original_e))
         return
 
 
@@ -91,19 +93,20 @@ class ReplyWithContentButNoText(Exception):
 
 
 class Fetcher():
+    browser_connection_is_custom = None
+    browser_connection_url = None
     browser_steps = None
     browser_steps_screenshot_path = None
     content = None
     error = None
     fetcher_description = "No description"
-    browser_connection_url = None
     headers = {}
+    instock_data = None
+    instock_data_js = ""
     status_code = None
     webdriver_js_execute_code = None
     xpath_data = None
     xpath_element_js = ""
-    instock_data = None
-    instock_data_js = ""
 
     # Will be needed in the future by the VisualSelector, always get this where possible.
     screenshot = False
@@ -172,7 +175,7 @@ class Fetcher():
 
     def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-        from playwright._impl._errors import TimeoutError
+        from playwright._impl._errors import TimeoutError, Error
         from jinja2 import Environment
         jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
 
@@ -202,10 +205,10 @@ class Fetcher():
                                                       optional_value=optional_value)
                     self.screenshot_step(step_n)
                     self.save_step_html(step_n)
-                except TimeoutError as e:
-                    print(str(e))
+
+                except (Error, TimeoutError) as e:
                     # Stop processing here
-                    raise BrowserStepsStepTimout(step_n=step_n)
+                    raise BrowserStepsStepException(step_n=step_n, original_e=e)
 
     # It's always good to reset these
     def delete_browser_steps_screenshots(self):
@@ -252,16 +255,19 @@ class base_html_playwright(Fetcher):
 
     proxy = None
 
-    def __init__(self, proxy_override=None, browser_connection_url=None):
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
         super().__init__()
 
         self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
 
-        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-        if not browser_connection_url:
-            self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
+        if custom_browser_connection_url:
+            self.browser_connection_is_custom = True
+            self.browser_connection_url = custom_browser_connection_url
         else:
-            self.browser_connection_url = browser_connection_url
+            # Fallback to fetching from system
+            # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+            self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
+
 
         # If any proxy settings are enabled, then we should setup the proxy object
         proxy_args = {}
@@ -421,8 +427,10 @@ class base_html_playwright(Fetcher):
             current_include_filters=None,
             is_binary=False):
 
+
         # For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!)
-        if not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
+        # browser_connection_is_custom doesnt work with puppeteer style fetch (use playwright native too in this case)
+        if not self.browser_connection_is_custom and not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
             if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')):
                 # Temporary backup solution until we rewrite the playwright code
                 return self.run_fetch_browserless_puppeteer(
@@ -569,15 +577,16 @@ class base_html_webdriver(Fetcher):
                                         'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
     proxy = None
 
-    def __init__(self, proxy_override=None, browser_connection_url=None):
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
         super().__init__()
         from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
 
         # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-        if not browser_connection_url:
+        if not custom_browser_connection_url:
             self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
         else:
-            self.browser_connection_url = browser_connection_url
+            self.browser_connection_is_custom = True
+            self.browser_connection_url = custom_browser_connection_url
 
         # If any proxy settings are enabled, then we should setup the proxy object
         proxy_args = {}
@@ -674,7 +683,7 @@ class base_html_webdriver(Fetcher):
 class html_requests(Fetcher):
     fetcher_description = "Basic fast Plaintext/HTTP Client"
 
-    def __init__(self, proxy_override=None, browser_connection_url=None):
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
         super().__init__()
         self.proxy_override = proxy_override
         # browser_connection_url is none because its always 'launched locally'

changedetectionio/model/Watch.py

@@ -38,6 +38,7 @@ base_config = {
     'track_ldjson_price_data': None,
     'headers': {},  # Extra headers to send
     'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
+    'in_stock' : None,
     'in_stock_only' : True, # Only trigger change on going to instock from out-of-stock
     'include_filters': [],
     'last_checked': 0,

changedetectionio/notification.py

@@ -46,6 +46,9 @@ from apprise.decorators import notify
 @notify(on="puts")
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     import requests
+    from apprise.utils import parse_url as apprise_parse_url
+    from apprise.URLBase import URLBase
+
     url = kwargs['meta'].get('url')
 
     if url.startswith('post'):
@@ -68,16 +71,45 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     url = url.replace('delete://', 'http://')
     url = url.replace('deletes://', 'https://')
 
-    # Try to auto-guess if it's JSON
     headers = {}
+    params = {}
+    auth = None
+
+    # Convert /foobar?+some-header=hello to proper header dictionary
+    results = apprise_parse_url(url)
+    if results:
+        # Add our headers that the user can potentially over-ride if they wish
+        # to to our returned result set and tidy entries by unquoting them
+        headers = {URLBase.unquote(x): URLBase.unquote(y)
+                   for x, y in results['qsd+'].items()}
+
+        # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+        # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
+        # but here we are making straight requests, so we need todo convert this against apprise's logic
+        for k, v in results['qsd'].items():
+            if not k.strip('+-') in results['qsd+'].keys():
+                params[URLBase.unquote(k)] = URLBase.unquote(v)
+
+        # Determine Authentication
+        auth = ''
+        if results.get('user') and results.get('password'):
+            auth = (URLBase.unquote(results.get('user')), URLBase.unquote(results.get('user')))
+        elif results.get('user'):
+            auth = (URLBase.unquote(results.get('user')))
+
+    # Try to auto-guess if it's JSON
     try:
         json.loads(body)
-        headers = {'Content-Type': 'application/json; charset=utf-8'}
+        headers['Content-Type'] = 'application/json; charset=utf-8'
     except ValueError as e:
         pass
 
-
-    r(url, headers=headers, data=body)
+    r(results.get('url'),
+      auth=auth,
+      data=body,
+      headers=headers,
+      params=params
+      )
 
 
 def process_notification(n_object, datastore):
@@ -87,8 +119,8 @@ def process_notification(n_object, datastore):
 
     # Get the notification body from datastore
     jinja2_env = Environment(loader=BaseLoader)
-    n_body = jinja2_env.from_string(n_object.get('notification_body', default_notification_body)).render(**notification_parameters)
-    n_title = jinja2_env.from_string(n_object.get('notification_title', default_notification_title)).render(**notification_parameters)
+    n_body = jinja2_env.from_string(n_object.get('notification_body', '')).render(**notification_parameters)
+    n_title = jinja2_env.from_string(n_object.get('notification_title', '')).render(**notification_parameters)
     n_format = valid_notification_formats.get(
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
@@ -155,8 +187,8 @@ def process_notification(n_object, datastore):
                     if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
                         prefix = '?' if not '?' in url else '&'
                         # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
-                        n_format = n_format.tolower()
-                        url = "{}{}format={}".format(url, prefix, n_format)
+                        n_format = n_format.lower()
+                        url = f"{url}{prefix}format={n_format}"
                     # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
 
                 apobj.add(url)
@@ -189,13 +221,14 @@ def process_notification(n_object, datastore):
 
 
 # Notification title + body content parameters get created here.
+# ( Where we prepare the tokens in the notification to be replaced with actual values )
 def create_notification_parameters(n_object, datastore):
     from copy import deepcopy
 
     # in the case we send a test notification from the main settings, there is no UUID.
     uuid = n_object['uuid'] if 'uuid' in n_object else ''
 
-    if uuid != '':
+    if uuid:
         watch_title = datastore.data['watching'][uuid].get('title', '')
         tag_list = []
         tags = datastore.get_all_tags_for_watch(uuid)
@@ -223,7 +256,7 @@ def create_notification_parameters(n_object, datastore):
     tokens.update(
         {
             'base_url': base_url,
-            'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '',
+            'current_snapshot': n_object.get('current_snapshot', ''),
             'diff': n_object.get('diff', ''),  # Null default in the case we use a test
             'diff_added': n_object.get('diff_added', ''),  # Null default in the case we use a test
             'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test

changedetectionio/processors/__init__.py

@@ -43,15 +43,20 @@ class difference_detection_processor():
 
         # In the case that the preferred fetcher was a browser config with custom connection URL..
         # @todo - on save watch, if its extra_browser_ then it should be obvious it will use playwright (like if its requests now..)
-        browser_connection_url = None
+        custom_browser_connection_url = None
         if prefer_fetch_backend.startswith('extra_browser_'):
             (t, key) = prefer_fetch_backend.split('extra_browser_')
             connection = list(
                 filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', [])))
             if connection:
                 prefer_fetch_backend = 'base_html_playwright'
-                browser_connection_url = connection[0].get('browser_connection_url')
+                custom_browser_connection_url = connection[0].get('browser_connection_url')
 
+        # PDF should be html_requests because playwright will serve it up (so far) in a embedded page
+        # @todo https://github.com/dgtlmoon/changedetection.io/issues/2019
+        # @todo needs test to or a fix
+        if self.watch.is_pdf:
+           prefer_fetch_backend = "html_requests"
 
         # Grab the right kind of 'fetcher', (playwright, requests, etc)
         if hasattr(content_fetcher, prefer_fetch_backend):
@@ -69,7 +74,7 @@ class difference_detection_processor():
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
         self.fetcher = fetcher_obj(proxy_override=proxy_url,
-                                   browser_connection_url=browser_connection_url
+                                   custom_browser_connection_url=custom_browser_connection_url
                                    )
 
         if self.watch.has_browser_steps:

changedetectionio/res/stock-not-in-stock.js

@@ -4,6 +4,7 @@ function isItemInStock() {
     ' أخبرني عندما يتوفر',
     '0 in stock',
     'agotado',
+    'article épuisé',
     'artikel zurzeit vergriffen',
     'as soon as stock is available',
     'ausverkauft', // sold out
@@ -17,10 +18,8 @@ function isItemInStock() {
     'currently have any tickets for this',
     'currently unavailable',
     'dostępne wkrótce',
-    'dostępne wkrótce',
     'en rupture de stock',
     'ist derzeit nicht auf lager',
-    'ist derzeit nicht auf lager',
     'item is no longer available',
     'let me know when it\'s available',
     'message if back in stock',
@@ -28,6 +27,8 @@ function isItemInStock() {
     'nicht auf lager',
     'nicht lieferbar',
     'nicht zur verfügung',
+    'niet beschikbaar',
+    'niet leverbaar',
     'no disponible temporalmente',
     'no longer in stock',
     'no tickets available',
@@ -40,9 +41,11 @@ function isItemInStock() {
     'out-of-stock',
     'produkt niedostępny',
     'sold out',
+    'sold-out',
     'temporarily out of stock',
     'temporarily unavailable',
     'tickets unavailable',
+    'tijdelijk uitverkocht',
     'unavailable tickets',
     'we do not currently have an estimate of when this product will be back in stock.',
     'zur zeit nicht an lager',
@@ -111,4 +114,4 @@ function isItemInStock() {
 }
 
 // returns the element text that makes it think it's out of stock
-return isItemInStock();
+return isItemInStock();

changedetectionio/static/js/notifications.js

@@ -24,14 +24,17 @@ $(document).ready(function() {
     })
 
     data = {
-        window_url : window.location.href,
-        notification_urls : $('.notification-urls').val(),
+      notification_body: $('#notification_body').val(),
+      notification_format: $('#notification_format').val(),
+      notification_title: $('#notification_title').val(),
+      notification_urls: $('.notification-urls').val(),
+      window_url: window.location.href,
     }
-    for (key in data) {
-      if (!data[key].length) {
-        alert(key+" is empty, cannot send test.")
-        return;
-      }
+
+
+    if (!data['notification_urls'].length) {
+      alert("Notification URL list is empty, cannot send test.")
+      return;
     }
 
     $.ajax({

changedetectionio/static/js/vis.js

@@ -2,21 +2,28 @@ $(document).ready(function () {
 
     // Lazy Hide/Show elements mechanism
     $('[data-visible-for]').hide();
-    $(':radio').on('keyup keypress blur change click', function (e) {
-        $('[data-visible-for]').hide();
-        $('.advanced-options').hide();
-        var n = $(this).attr('name') + "=" + $(this).val();
+    function show_related_elem(e) {
+        var n = $(e).attr('name') + "=" + $(e).val();
         if (n === 'fetch_backend=system') {
             n = "fetch_backend=" + default_system_fetch_backend;
         }
         $(`[data-visible-for~="${n}"]`).show();
-
+    }
+    $(':radio').on('keyup keypress blur change click', function (e) {
+        $(`[data-visible-for]`).hide();
+        $('.advanced-options').hide();
+        show_related_elem(this);
     });
-    $(':radio:checked').change();
+
+    $(':radio:checked').each(function (e) {
+       show_related_elem(this);
+    })
 
 
     // Show advanced
     $('.show-advanced').click(function (e) {
-        $(this).closest('.tab-pane-inner').find('.advanced-options').toggle();
+        $(this).closest('.tab-pane-inner').find('.advanced-options').each(function (e) {
+            $(this).toggle();
+        })
     });
 });

changedetectionio/static/js/watch-settings.js

@@ -1,18 +1,4 @@
 $(document).ready(function () {
-
-    // Lazy Hide/Show elements mechanism
-    $('[data-visible-for]').hide();
-    $(':radio').on('keyup keypress blur change click', function (e){
-        $('[data-visible-for]').hide();
-        var n = $(this).attr('name') + "=" + $(this).val();
-        if (n === 'fetch_backend=system') {
-            n = "fetch_backend=" + default_system_fetch_backend;
-        }
-        $(`[data-visible-for~="${n}"]`).show();
-
-    });
-    $(':radio:checked').change();
-
     $('#notification-setting-reset-to-default').click(function (e) {
         $('#notification_title').val('');
         $('#notification_body').val('');

changedetectionio/store.py

@@ -234,7 +234,7 @@ class ChangeDetectionStore:
 
         # Probably their should be dict...
         for watch in self.data['watching'].values():
-            if watch['url'] == url:
+            if watch['url'].lower() == url.lower():
                 return True
 
         return False
@@ -248,6 +248,7 @@ class ChangeDetectionStore:
                 'check_count': 0,
                 'fetch_time' : 0.0,
                 'has_ldjson_price_data': None,
+                'in_stock': None,
                 'last_checked': 0,
                 'last_error': False,
                 'last_notification_error': False,
@@ -333,7 +334,8 @@ class ChangeDetectionStore:
 
         # Or if UUIDs given directly
         if tag_uuids:
-            apply_extras['tags'] = list(set(apply_extras['tags'] + tag_uuids))
+            for t in tag_uuids:
+                apply_extras['tags'] = list(set(apply_extras['tags'] + [t.strip()]))
 
         # Make any uuids unique
         if apply_extras.get('tags'):

changedetectionio/templates/_common_fields.jinja

@@ -16,7 +16,7 @@
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
-                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
+                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>) <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes#postposts">more help here</a></li>
                                   <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                               </ul>
                             </div>

changedetectionio/templates/edit.html

@@ -14,7 +14,7 @@
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
 {% endif %}
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
     const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
     const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
     const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
@@ -127,7 +127,7 @@
                 {% endif %}
 
                 <!-- webdriver always -->
-                <fieldset data-visible-for="fetch_backend=html_webdriver">
+                <fieldset data-visible-for="fetch_backend=html_webdriver"  style="display: none;">
                     <div class="pure-control-group">
                         {{ render_field(form.webdriver_delay) }}
                         <div class="pure-form-message-inline">
@@ -153,15 +153,15 @@
                     </div>
                 </fieldset>
                 <!-- html requests always -->
-                <fieldset data-visible-for="fetch_backend=html_requests" style="display: none;">
+                <fieldset data-visible-for="fetch_backend=html_requests">
                     <div class="pure-control-group">
                         <a class="pure-button button-secondary button-xsmall show-advanced">Show advanced options</a>
                     </div>
                     <div class="advanced-options"  style="display: none;">
-                        <div class="pure-control-group advanced-options" id="request-method" style="display: none;">
+                        <div class="pure-control-group" id="request-method">
                             {{ render_field(form.method) }}
                         </div>
-                        <div class="advanced-options" id="request-body">
+                        <div id="request-body">
                                             {{ render_field(form.body, rows=5, placeholder="Example
 {
    \"name\":\"John\",
@@ -187,8 +187,8 @@ User-Agent: wonderbra 1.0") }}
                             (Not supported by Selenium browser)
                         </div>
                     </div>
-            <fieldset data-visible-for="fetch_backend=html_requests fetch_backend=html_webdriver" style="display: none;">
-                    <div  class="pure-control-group inline-radio advanced-options"  style="display: none;">
+            <fieldset data-visible-for="fetch_backend=html_requests fetch_backend=html_webdriver" >
+                    <div class="pure-control-group inline-radio advanced-options"  style="display: none;">
                     {{ render_checkbox_field(form.ignore_status_codes) }}
                     </div>
             </fieldset>

changedetectionio/templates/settings.html

@@ -4,7 +4,7 @@
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
 {% endif %}
@@ -236,8 +236,11 @@ nav
                 <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
                 </div>
                 <div class="pure-control-group" id="extra-browsers-setting">
-                    <span class="pure-form-message-inline"><i>Extra Browsers</i> allow changedetection.io to communicate with a different web-browser.</span><br>
-                  {{ render_field(form.requests.form.extra_browsers) }}
+                    <p>
+                    <span class="pure-form-message-inline"><i>Extra Browsers</i> can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.</span><br>
+                    <span class="pure-form-message-inline">Simply paste the connection address into the box, <a href="https://changedetection.io/tutorial/using-bright-datas-scraping-browser-pass-captchas-and-other-protection-when-monitoring">More instructions and examples here</a> </span>
+                    </p>
+                    {{ render_field(form.requests.form.extra_browsers) }}
                 </div>
             </div>
             <div id="actions">

changedetectionio/templates/watch-overview.html

@@ -141,7 +141,7 @@
                     {% if watch['processor'] == 'restock_diff'  %}
                     <span class="restock-label {{'in-stock' if watch['in_stock'] else 'not-in-stock' }}" title="detecting restock conditions">
                         <!-- maybe some object watch['processor'][restock_diff] or.. -->
-                        {% if watch['last_checked'] %}
+                        {% if watch['last_checked'] and watch['in_stock'] != None %}
                             {% if watch['in_stock'] %} In stock {% else %} Not in stock {% endif %}
                         {% else %}
                             Not yet checked

changedetectionio/tests/conftest.py

@@ -13,22 +13,17 @@ global app
 
 
 def cleanup(datastore_path):
+    import glob
     # Unlink test output files
-    files = [
-        'count.txt',
-        'endpoint-content.txt'
-        'headers.txt',
-        'headers-testtag.txt',
-        'notification.txt',
-        'secret.txt',
-        'url-watches.json',
-        'output.txt',
-    ]
-    for file in files:
-        try:
-            os.unlink("{}/{}".format(datastore_path, file))
-        except FileNotFoundError:
-            pass
+
+    for g in ["*.txt", "*.json", "*.pdf"]:
+        files = glob.glob(os.path.join(datastore_path, g))
+        for f in files:
+            if 'proxies.json' in f:
+                # Usually mounted by docker container during test time
+                continue
+            if os.path.isfile(f):
+                os.unlink(f)
 
 @pytest.fixture(scope='session')
 def app(request):

changedetectionio/tests/test_api.py

@@ -357,3 +357,25 @@ def test_api_watch_PUT_update(client, live_server):
     # Cleanup everything
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
+
+
+def test_api_import(client, live_server):
+    api_key = extract_api_key_from_UI(client)
+
+    res = client.post(
+        url_for("import") + "?tag=import-test",
+        data='https://website1.com\r\nhttps://website2.com',
+        headers={'x-api-key': api_key},
+        follow_redirects=True
+    )
+
+    assert res.status_code == 200
+    assert len(res.json) == 2
+    res = client.get(url_for("index"))
+    assert b"https://website1.com" in res.data
+    assert b"https://website2.com" in res.data
+
+    # Should see the new tag in the tag/groups list
+    res = client.get(url_for('tags.tags_overview_page'))
+    assert b'import-test' in res.data
+    

changedetectionio/tests/test_notification.py

@@ -281,7 +281,8 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
 
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}"
+    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
 
     res = client.post(
         url_for("settings_page"),
@@ -297,10 +298,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         follow_redirects=True
     )
     assert b'Settings updated' in res.data
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )
+
     # Add a watch and trigger a HTTP POST
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -315,7 +313,9 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     set_modified_response()
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
+    wait_for_all_checks(client)
+
+    time.sleep(2) # plus extra delay for notifications to fire
 
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
@@ -328,6 +328,13 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     with open("test-datastore/notification-url.txt", 'r') as f:
         notification_url = f.read()
         assert 'xxx=http' in notification_url
+        # apprise style headers should be stripped
+        assert 'custom-header' not in notification_url
+
+    with open("test-datastore/notification-headers.txt", 'r') as f:
+        notification_headers = f.read()
+        assert 'custom-header: 123' in notification_headers.lower()
+
 
     # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
     assert os.path.isfile("test-datastore/notification-content-type.txt")
@@ -335,3 +342,8 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         assert 'application/json' in f.read()
 
     os.unlink("test-datastore/notification-url.txt")
+
+    client.get(
+        url_for("form_delete", uuid="all"),
+        follow_redirects=True
+    )

changedetectionio/tests/util.py

@@ -205,6 +205,9 @@ def live_server_setup(live_server):
         with open("test-datastore/notification-url.txt", "w") as f:
             f.write(request.url)
 
+        with open("test-datastore/notification-headers.txt", "w") as f:
+            f.write(str(request.headers))
+
         if request.content_type:
             with open("test-datastore/notification-content-type.txt", "w") as f:
                 f.write(request.content_type)

changedetectionio/update_worker.py

@@ -26,47 +26,61 @@ class update_worker(threading.Thread):
         self.datastore = datastore
         super().__init__(*args, **kwargs)
 
-    def queue_notification_for_watch(self, n_object, watch):
+    def queue_notification_for_watch(self, notification_q, n_object, watch):
 
         from changedetectionio import diff
+        dates = []
+        trigger_text = ''
+
+        if watch:
+            watch_history = watch.history
+            dates = list(watch_history.keys())
+            trigger_text = watch.get('trigger_text', [])
 
-        watch_history = watch.history
-        dates = list(watch_history.keys())
         # Add text that was triggered
-        snapshot_contents = watch.get_history_snapshot(dates[-1])
+        if len(dates):
+            snapshot_contents = watch.get_history_snapshot(dates[-1])
+        else:
+            snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
 
         # HTML needs linebreak, but MarkDown and Text can use a linefeed
-        if n_object['notification_format'] == 'HTML':
+        if n_object.get('notification_format') == 'HTML':
             line_feed_sep = "<br>"
             # Snapshot will be plaintext on the disk, convert to some kind of HTML
             snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
         else:
             line_feed_sep = "\n"
 
-        trigger_text = watch.get('trigger_text', [])
         triggered_text = ''
-
         if len(trigger_text):
             from . import html_tools
             triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
             if triggered_text:
                 triggered_text = line_feed_sep.join(triggered_text)
 
+        # Could be called as a 'test notification' with only 1 snapshot available
+        prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
+        current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
+
+        if len(dates) > 1:
+            prev_snapshot = watch.get_history_snapshot(dates[-2])
+            current_snapshot = watch.get_history_snapshot(dates[-1])
 
         n_object.update({
             'current_snapshot': snapshot_contents,
-            'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
-            'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
-            'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
-            'diff_patch': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep, patch_format=True),
-            'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
-            'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
+            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep),
+            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
+            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep),
+            'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
+            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
+            'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
             'triggered_text': triggered_text,
-            'uuid': watch.get('uuid'),
-            'watch_url': watch.get('url'),
+            'uuid': watch.get('uuid') if watch else None,
+            'watch_url': watch.get('url') if watch else None,
         })
-        logging.info (">> SENDING NOTIFICATION")
-        self.notification_q.put(n_object)
+        logging.info(">> SENDING NOTIFICATION")
+
+        notification_q.put(n_object)
 
     # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
     def _check_cascading_vars(self, var_name, watch):
@@ -134,7 +148,7 @@ class update_worker(threading.Thread):
         queued = False
         if n_object and n_object.get('notification_urls'):
             queued = True
-            self.queue_notification_for_watch(n_object, watch)
+            self.queue_notification_for_watch(notification_q=self.notification_q, n_object=n_object, watch=watch)
 
         return queued
 
@@ -174,9 +188,9 @@ class update_worker(threading.Thread):
             return
         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
         n_object = {'notification_title': "Changedetection.io - Alert - Browser step at position {} could not be run".format(step_n+1),
-                    'notification_body': "Your configured browser step at position {} for {{watch['url']}} "
+                    'notification_body': "Your configured browser step at position {} for {{{{watch_url}}}} "
                                          "did not appear on the page after {} attempts, did the page change layout? "
-                                         "Does it need a delay added?\n\nLink: {{base_url}}/edit/{{watch_uuid}}\n\n"
+                                         "Does it need a delay added?\n\nLink: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\n"
                                          "Thanks - Your omniscient changedetection.io installation :)\n".format(step_n+1, threshold),
                     'notification_format': 'text'}
 
@@ -340,20 +354,32 @@ class update_worker(threading.Thread):
                         changed_detected = False
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False})
 
-                    except content_fetcher.BrowserStepsStepTimout as e:
+                    except content_fetcher.BrowserStepsStepException as e:
 
                         if not self.datastore.data['watching'].get(uuid):
                             continue
 
                         error_step = e.step_n + 1
-                        err_text = f"Warning, browser step at position {error_step} could not run, target not found, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step"
+                        from playwright._impl._errors import TimeoutError, Error
+
+                        # Generally enough info for TimeoutError (couldnt locate the element after default seconds)
+                        err_text = f"Browser step at position {error_step} could not run, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step."
+
+                        if e.original_e.name == "TimeoutError":
+                            # Just the first line is enough, the rest is the stack trace
+                            err_text += " Could not find the target."
+                        else:
+                            # Other Error, more info is good.
+                            err_text += " " + str(e.original_e).splitlines()[0]
+
+                        print(f"BrowserSteps exception at step {error_step}", str(e.original_e))
+
                         self.datastore.update_watch(uuid=uuid,
                                                     update_obj={'last_error': err_text,
                                                                 'browser_steps_last_error_step': error_step
                                                                 }
                                                     )
 
-
                         if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
                             c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
                             c += 1

docker-compose.yml

@@ -90,7 +90,7 @@ services:
 #
 
      # Used for fetching pages via Playwright+Chrome where you need Javascript support.
-     # Note: works well but is deprecated, doesnt fetch full page screenshots and other issues
+     # Note: works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector) and other issues
 #    browser-chrome:
 #        hostname: browser-chrome
 #        image: selenium/standalone-chrome:4

docs/api_v1/index.html

@@ -5,13 +5,13 @@
   <meta name="description" content="Manage your changedetection.io watches via API, requires the `x-api-key` header which is found in the settings UI.">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-  <link href="assets/bootstrap.min.css?v=1677105736053" rel="stylesheet" media="screen">
-  <link href="assets/prism.css?v=1677105736053" rel="stylesheet" />
-  <link href="assets/main.css?v=1677105736053" rel="stylesheet" media="screen, print">
-  <link href="assets/favicon.ico?v=1677105736053" rel="icon" type="image/x-icon">
-  <link href="assets/apple-touch-icon.png?v=1677105736053" rel="apple-touch-icon" sizes="180x180">
-  <link href="assets/favicon-32x32.png?v=1677105736053" rel="icon" type="image/png" sizes="32x32">
-  <link href="assets/favicon-16x16.png?v=1677105736053" rel="icon" type="image/png" sizes="16x16">
+  <link href="assets/bootstrap.min.css?v=1701595483622" rel="stylesheet" media="screen">
+  <link href="assets/prism.css?v=1701595483622" rel="stylesheet" />
+  <link href="assets/main.css?v=1701595483622" rel="stylesheet" media="screen, print">
+  <link href="assets/favicon.ico?v=1701595483622" rel="icon" type="image/x-icon">
+  <link href="assets/apple-touch-icon.png?v=1701595483622" rel="apple-touch-icon" sizes="180x180">
+  <link href="assets/favicon-32x32.png?v=1701595483622" rel="icon" type="image/png" sizes="32x32">
+  <link href="assets/favicon-16x16.png?v=1701595483622" rel="icon" type="image/png" sizes="16x16">
 </head>
 
 <body class="container-fluid">
@@ -928,6 +928,6 @@
   </div>
 </div>
 
-<script src="assets/main.bundle.js?v=1677105736053"></script>
+<script src="assets/main.bundle.js?v=1701595483622"></script>
 </body>
 </html>

heroku.yml

@@ -1,2 +0,0 @@
-run:
-  changedetection: python3 ./changedetection.py -C -d ./datastore -p $PORT

requirements.txt

@@ -30,7 +30,7 @@ dnspython~=2.4 # related to eventlet fixes
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise~=1.6.0
+apprise~=1.7.1
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt

setup.py

@@ -27,12 +27,12 @@ install_requires = open('requirements.txt').readlines()
 setup(
     name='changedetection.io',
     version=find_version("changedetectionio", "__init__.py"),
-    description='Website change detection and monitoring service',
+    description='Website change detection and monitoring service, detect changes to web pages and send alerts/notifications.',
     long_description=open('README-pip.md').read(),
     long_description_content_type='text/markdown',
     keywords='website change monitor for changes notification change detection '
              'alerts tracking website tracker change alert website and monitoring',
-    entry_points={"console_scripts": ["changedetection.io=changedetectionio.changedetection:main"]},
+    entry_points={"console_scripts": ["changedetection.io=changedetectionio:main"]},
     zip_safe=True,
     scripts=["changedetection.py"],
     author='dgtlmoon',
@@ -41,7 +41,7 @@ setup(
     include_package_data=True,
     install_requires=install_requires,
     license="Apache License 2.0",
-    python_requires=">= 3.7",
+    python_requires=">= 3.10",
     classifiers=['Intended Audience :: Customer Service',
                  'Intended Audience :: Developers',
                  'Intended Audience :: Education',