"In stock" should be 'none' by default

.github/workflows/test-only.yml

@@ -98,14 +98,58 @@ jobs:
 
       - name: Test changedetection.io container starts+runs basically without error
         run: |
-          docker run -p 5556:5000 -d test-changedetectionio
+          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
           sleep 3
           # Should return 0 (no error) when grep finds it
           curl -s http://localhost:5556 |grep -q checkbox-uuid
           
           # and IPv6
           curl -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+          docker kill test-changedetectionio
+
+      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
+        run: |
           
+          echo SIGINT Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGINT to sig-test container"
+          docker kill --signal=SIGINT sig-test
+          sleep 3
+          # invert the check (it should be not 0/not running)
+          docker ps
+          # check signal catch(STDOUT) log
+          docker logs sig-test | grep 'Shutdown: Got Signal - SIGINT' || exit 1
+          test -z "`docker ps|grep sig-test`"
+          if [ $? -ne 0 ]
+          then
+            echo "Looks like container was running when it shouldnt be"
+            docker ps
+            exit 1
+          fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists 
+          docker rm sig-test
+          
+          echo SIGTERM Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGTERM to sig-test container"
+          docker kill --signal=SIGTERM sig-test
+          sleep 3
+          # invert the check (it should be not 0/not running)
+          docker ps
+          docker logs sig-test | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
+          test -z "`docker ps|grep sig-test`"
+          if [ $? -ne 0 ]
+          then
+            echo "Looks like container was running when it shouldnt be"
+            docker ps
+            exit 1
+          fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists           
+          docker rm sig-test
 
 #export WEBDRIVER_URL=http://localhost:4444/wd/hub
 #pytest tests/fetchers/test_content.py

Dockerfile

@@ -53,12 +53,12 @@ ENV PYTHONPATH=/usr/local
 
 EXPOSE 5000
 
-# The actual flask app
+# The actual flask app module
 COPY changedetectionio /app/changedetectionio
-
-# The eventlet server wrapper
+# Starting wrapper
 COPY changedetection.py /app/changedetection.py
 
 WORKDIR /app
+CMD ["python", "./changedetection.py", "-d", "/datastore"]
+
 
-CMD [ "python", "./changedetection.py" , "-d", "/datastore"]

README.md

@@ -11,9 +11,10 @@ _Live your data-life pro-actively._
 
 ![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
 
-[**Don't have time? Let us host it for you! try our $8.99/month subscription - use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_
+[**Get started with website page change monitoring straight away. Don't have time? Try our $8.99/month subscription, use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_
 
 - Chrome browser included.
+- Nothing to install, access via browser login after signup.
 - Super fast, no registration needed setup.
 - Get started watching and receiving website change notifications straight away.
 

changedetection.py

@@ -1,44 +1,6 @@
 #!/usr/bin/python3
 
-# Entry-point for running from the CLI when not installed via Pip, Pip will handle the console_scripts entry_points's from setup.py
-# It's recommended to use `pip3 install changedetection.io` and start with `changedetection.py` instead, it will be linkd to your global path.
-# or Docker.
-# Read more https://github.com/dgtlmoon/changedetection.io/wiki
+# Only exists for direct CLI usage
 
-from changedetectionio import changedetection
-import multiprocessing
-import sys
-import os
-
-def sigchld_handler(_signo, _stack_frame):
-    import sys
-    print('Shutdown: Got SIGCHLD')
-    # https://stackoverflow.com/questions/40453496/python-multiprocessing-capturing-signals-to-restart-child-processes-or-shut-do
-    pid, status = os.waitpid(-1, os.WNOHANG | os.WUNTRACED | os.WCONTINUED)
-
-    print('Sub-process: pid %d status %d' % (pid, status))
-    if status != 0:
-        sys.exit(1)
-
-    raise SystemExit
-
-if __name__ == '__main__':
-
-    #signal.signal(signal.SIGCHLD, sigchld_handler)
-
-    # The only way I could find to get Flask to shutdown, is to wrap it and then rely on the subsystem issuing SIGTERM/SIGKILL
-    parse_process = multiprocessing.Process(target=changedetection.main)
-    parse_process.daemon = True
-    parse_process.start()
-    import time
-
-    try:
-        while True:
-            time.sleep(1)
-            if not parse_process.is_alive():
-                # Process died/crashed for some reason, exit with error set
-                sys.exit(1)
-
-    except KeyboardInterrupt:
-        #parse_process.terminate() not needed, because this process will issue it to the sub-process anyway
-        print ("Exited - CTRL+C")
+import changedetectionio
+changedetectionio.main()

changedetectionio/api/api_v1.py

@@ -296,6 +296,61 @@ class CreateWatch(Resource):
 
         return list, 200
 
+class Import(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    def post(self):
+        """
+        @api {post} /api/v1/import Import a list of watched URLs
+        @apiDescription Accepts a line-feed separated list of URLs to import, additionally with ?tag_uuids=(tag  id), ?tag=(name), ?proxy={key}, ?dedupe=true (default true) one URL per line.
+        @apiExample {curl} Example usage:
+            curl http://localhost:5000/api/v1/import --data-binary @list-of-sites.txt -H"x-api-key:8a111a21bc2f8f1dd9b9353bbd46049a"
+        @apiName Import
+        @apiGroup Watch
+        @apiSuccess (200) {List} OK List of watch UUIDs added
+        @apiSuccess (500) {String} ERR Some other error
+        """
+
+        extras = {}
+
+        if request.args.get('proxy'):
+            plist = self.datastore.proxy_list
+            if not request.args.get('proxy') in plist:
+                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
+            else:
+                extras['proxy'] = request.args.get('proxy')
+
+        dedupe = strtobool(request.args.get('dedupe', 'true'))
+
+        tags = request.args.get('tag')
+        tag_uuids = request.args.get('tag_uuids')
+
+        if tag_uuids:
+            tag_uuids = tag_uuids.split(',')
+
+        urls = request.get_data().decode('utf8').splitlines()
+        added = []
+        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
+        for url in urls:
+            url = url.strip()
+            if not len(url):
+                continue
+
+            # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+            if not validators.url(url, simple_host=allow_simplehost):
+                return f"Invalid or unsupported URL - {url}", 400
+
+            if dedupe and self.datastore.url_exists(url):
+                continue
+
+            new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags, tag_uuids=tag_uuids)
+            added.append(new_uuid)
+
+        return added
+
 class SystemInfo(Resource):
     def __init__(self, **kwargs):
         # datastore is a black box dependency

changedetectionio/blueprint/browser_steps/__init__.py

@@ -25,10 +25,9 @@ from distutils.util import strtobool
 from flask import Blueprint, request, make_response
 import logging
 import os
-import re
 
 from changedetectionio.store import ChangeDetectionStore
-from changedetectionio import login_optionally_required
+from changedetectionio.flask_app import login_optionally_required
 
 browsersteps_sessions = {}
 io_interface_context = None

changedetectionio/blueprint/tags/__init__.py

@@ -1,6 +1,6 @@
 from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
 from changedetectionio.store import ChangeDetectionStore
-from changedetectionio import login_optionally_required
+from changedetectionio.flask_app import login_optionally_required
 
 
 def construct_blueprint(datastore: ChangeDetectionStore):

changedetectionio/changedetection.py

@@ -1,153 +0,0 @@
-#!/usr/bin/python3
-
-# Launch as a eventlet.wsgi server instance.
-
-from distutils.util import strtobool
-from json.decoder import JSONDecodeError
-
-import eventlet
-import eventlet.wsgi
-import getopt
-import os
-import signal
-import socket
-import sys
-
-from . import store, changedetection_app, content_fetcher
-from . import __version__
-
-# Only global so we can access it in the signal handler
-app = None
-datastore = None
-
-def sigterm_handler(_signo, _stack_frame):
-    global app
-    global datastore
-#    app.config.exit.set()
-    print('Shutdown: Got SIGTERM, DB saved to disk')
-    datastore.sync_to_json()
-#    raise SystemExit
-
-def main():
-    global datastore
-    global app
-
-    datastore_path = None
-    do_cleanup = False
-    host = ''
-    ipv6_enabled = False
-    port = os.environ.get('PORT') or 5000
-    ssl_mode = False
-
-    # On Windows, create and use a default path.
-    if os.name == 'nt':
-        datastore_path = os.path.expandvars(r'%APPDATA%\changedetection.io')
-        os.makedirs(datastore_path, exist_ok=True)
-    else:
-        # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
-        datastore_path = os.path.join(os.getcwd(), "../datastore")
-
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:", "port")
-    except getopt.GetoptError:
-        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
-        sys.exit(2)
-
-    create_datastore_dir = False
-
-    for opt, arg in opts:
-        if opt == '-s':
-            ssl_mode = True
-
-        if opt == '-h':
-            host = arg
-
-        if opt == '-p':
-            port = int(arg)
-
-        if opt == '-d':
-            datastore_path = arg
-
-        if opt == '-6':
-            print ("Enabling IPv6 listen support")
-            ipv6_enabled = True
-
-        # Cleanup (remove text files that arent in the index)
-        if opt == '-c':
-            do_cleanup = True
-
-        # Create the datadir if it doesnt exist
-        if opt == '-C':
-            create_datastore_dir = True
-
-    # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
-    app_config = {'datastore_path': datastore_path}
-
-    if not os.path.isdir(app_config['datastore_path']):
-        if create_datastore_dir:
-            os.mkdir(app_config['datastore_path'])
-        else:
-            print(
-                "ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists or specify a directory with the -d option.\n"
-                "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
-            sys.exit(2)
-
-    try:
-        datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
-    except JSONDecodeError as e:
-        # Dont' start if the JSON DB looks corrupt
-        print ("ERROR: JSON DB or Proxy List JSON at '{}' appears to be corrupt, aborting".format(app_config['datastore_path']))
-        print(str(e))
-        return
-
-    app = changedetection_app(app_config, datastore)
-
-    signal.signal(signal.SIGTERM, sigterm_handler)
-
-    # Go into cleanup mode
-    if do_cleanup:
-        datastore.remove_unused_snapshots()
-
-    app.config['datastore_path'] = datastore_path
-
-
-    @app.context_processor
-    def inject_version():
-        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
-                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
-                    has_password=datastore.data['settings']['application']['password'] != False
-                    )
-
-    # Monitored websites will not receive a Referer header when a user clicks on an outgoing link.
-    # @Note: Incompatible with password login (and maybe other features) for now, submit a PR!
-    @app.after_request
-    def hide_referrer(response):
-        if strtobool(os.getenv("HIDE_REFERER", 'false')):
-            response.headers["Referrer-Policy"] = "no-referrer"
-
-        return response
-
-    # Proxy sub-directory support
-    # Set environment var USE_X_SETTINGS=1 on this script
-    # And then in your proxy_pass settings
-    #
-    #         proxy_set_header Host "localhost";
-    #         proxy_set_header X-Forwarded-Prefix /app;
-
-    if os.getenv('USE_X_SETTINGS'):
-        print ("USE_X_SETTINGS is ENABLED\n")
-        from werkzeug.middleware.proxy_fix import ProxyFix
-        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
-
-    s_type = socket.AF_INET6 if ipv6_enabled else socket.AF_INET
-
-    if ssl_mode:
-        # @todo finalise SSL config, but this should get you in the right direction if you need it.
-        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port), s_type),
-                                               certfile='cert.pem',
-                                               keyfile='privkey.pem',
-                                               server_side=True), app)
-
-    else:
-        eventlet.wsgi.server(eventlet.listen((host, int(port)), s_type), app)
-

changedetectionio/model/Watch.py

@@ -38,6 +38,7 @@ base_config = {
     'track_ldjson_price_data': None,
     'headers': {},  # Extra headers to send
     'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
+    'in_stock' : None,
     'in_stock_only' : True, # Only trigger change on going to instock from out-of-stock
     'include_filters': [],
     'last_checked': 0,

changedetectionio/notification.py

@@ -46,6 +46,9 @@ from apprise.decorators import notify
 @notify(on="puts")
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     import requests
+    from apprise.utils import parse_url as apprise_parse_url
+    from apprise.URLBase import URLBase
+
     url = kwargs['meta'].get('url')
 
     if url.startswith('post'):
@@ -68,16 +71,45 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     url = url.replace('delete://', 'http://')
     url = url.replace('deletes://', 'https://')
 
-    # Try to auto-guess if it's JSON
     headers = {}
+    params = {}
+    auth = None
+
+    # Convert /foobar?+some-header=hello to proper header dictionary
+    results = apprise_parse_url(url)
+    if results:
+        # Add our headers that the user can potentially over-ride if they wish
+        # to to our returned result set and tidy entries by unquoting them
+        headers = {URLBase.unquote(x): URLBase.unquote(y)
+                   for x, y in results['qsd+'].items()}
+
+        # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+        # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
+        # but here we are making straight requests, so we need todo convert this against apprise's logic
+        for k, v in results['qsd'].items():
+            if not k.strip('+-') in results['qsd+'].keys():
+                params[URLBase.unquote(k)] = URLBase.unquote(v)
+
+        # Determine Authentication
+        auth = ''
+        if results.get('user') and results.get('password'):
+            auth = (URLBase.unquote(results.get('user')), URLBase.unquote(results.get('user')))
+        elif results.get('user'):
+            auth = (URLBase.unquote(results.get('user')))
+
+    # Try to auto-guess if it's JSON
     try:
         json.loads(body)
-        headers = {'Content-Type': 'application/json; charset=utf-8'}
+        headers['Content-Type'] = 'application/json; charset=utf-8'
     except ValueError as e:
         pass
 
-
-    r(url, headers=headers, data=body)
+    r(results.get('url'),
+      auth=auth,
+      data=body,
+      headers=headers,
+      params=params
+      )
 
 
 def process_notification(n_object, datastore):

changedetectionio/processors/__init__.py

@@ -52,6 +52,11 @@ class difference_detection_processor():
                 prefer_fetch_backend = 'base_html_playwright'
                 browser_connection_url = connection[0].get('browser_connection_url')
 
+        # PDF should be html_requests because playwright will serve it up (so far) in a embedded page
+        # @todo https://github.com/dgtlmoon/changedetection.io/issues/2019
+        # @todo needs test to or a fix
+        if self.watch.is_pdf:
+           prefer_fetch_backend = "html_requests"
 
         # Grab the right kind of 'fetcher', (playwright, requests, etc)
         if hasattr(content_fetcher, prefer_fetch_backend):

changedetectionio/res/stock-not-in-stock.js

@@ -28,6 +28,8 @@ function isItemInStock() {
     'nicht auf lager',
     'nicht lieferbar',
     'nicht zur verfügung',
+    'niet leverbaar',
+    'niet beschikbaar',
     'no disponible temporalmente',
     'no longer in stock',
     'no tickets available',
@@ -40,9 +42,11 @@ function isItemInStock() {
     'out-of-stock',
     'produkt niedostępny',
     'sold out',
+    'sold-out',
     'temporarily out of stock',
     'temporarily unavailable',
     'tickets unavailable',
+    'tijdelijk uitverkocht',
     'unavailable tickets',
     'we do not currently have an estimate of when this product will be back in stock.',
     'zur zeit nicht an lager',
@@ -111,4 +115,4 @@ function isItemInStock() {
 }
 
 // returns the element text that makes it think it's out of stock
-return isItemInStock();
+return isItemInStock();

changedetectionio/store.py

@@ -234,7 +234,7 @@ class ChangeDetectionStore:
 
         # Probably their should be dict...
         for watch in self.data['watching'].values():
-            if watch['url'] == url:
+            if watch['url'].lower() == url.lower():
                 return True
 
         return False
@@ -248,6 +248,7 @@ class ChangeDetectionStore:
                 'check_count': 0,
                 'fetch_time' : 0.0,
                 'has_ldjson_price_data': None,
+                'in_stock': None,
                 'last_checked': 0,
                 'last_error': False,
                 'last_notification_error': False,
@@ -333,7 +334,8 @@ class ChangeDetectionStore:
 
         # Or if UUIDs given directly
         if tag_uuids:
-            apply_extras['tags'] = list(set(apply_extras['tags'] + tag_uuids))
+            for t in tag_uuids:
+                apply_extras['tags'] = list(set(apply_extras['tags'] + [t.strip()]))
 
         # Make any uuids unique
         if apply_extras.get('tags'):

changedetectionio/templates/_common_fields.jinja

@@ -16,7 +16,7 @@
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
-                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
+                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>) <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes#postposts">more help here</a></li>
                                   <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                               </ul>
                             </div>

changedetectionio/templates/settings.html

@@ -236,8 +236,11 @@ nav
                 <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
                 </div>
                 <div class="pure-control-group" id="extra-browsers-setting">
-                    <span class="pure-form-message-inline"><i>Extra Browsers</i> allow changedetection.io to communicate with a different web-browser.</span><br>
-                  {{ render_field(form.requests.form.extra_browsers) }}
+                    <p>
+                    <span class="pure-form-message-inline"><i>Extra Browsers</i> can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.</span><br>
+                    <span class="pure-form-message-inline">Simply paste the connection address into the box, <a href="https://changedetection.io/tutorial/using-bright-datas-scraping-browser-pass-captchas-and-other-protection-when-monitoring">More instructions and examples here</a> </span>
+                    </p>
+                    {{ render_field(form.requests.form.extra_browsers) }}
                 </div>
             </div>
             <div id="actions">

changedetectionio/templates/watch-overview.html

@@ -141,7 +141,7 @@
                     {% if watch['processor'] == 'restock_diff'  %}
                     <span class="restock-label {{'in-stock' if watch['in_stock'] else 'not-in-stock' }}" title="detecting restock conditions">
                         <!-- maybe some object watch['processor'][restock_diff] or.. -->
-                        {% if watch['last_checked'] %}
+                        {% if watch['last_checked'] and watch['in_stock'] != None %}
                             {% if watch['in_stock'] %} In stock {% else %} Not in stock {% endif %}
                         {% else %}
                             Not yet checked

changedetectionio/tests/conftest.py

@@ -13,22 +13,17 @@ global app
 
 
 def cleanup(datastore_path):
+    import glob
     # Unlink test output files
-    files = [
-        'count.txt',
-        'endpoint-content.txt'
-        'headers.txt',
-        'headers-testtag.txt',
-        'notification.txt',
-        'secret.txt',
-        'url-watches.json',
-        'output.txt',
-    ]
-    for file in files:
-        try:
-            os.unlink("{}/{}".format(datastore_path, file))
-        except FileNotFoundError:
-            pass
+
+    for g in ["*.txt", "*.json", "*.pdf"]:
+        files = glob.glob(os.path.join(datastore_path, g))
+        for f in files:
+            if 'proxies.json' in f:
+                # Usually mounted by docker container during test time
+                continue
+            if os.path.isfile(f):
+                os.unlink(f)
 
 @pytest.fixture(scope='session')
 def app(request):

changedetectionio/tests/test_api.py

@@ -357,3 +357,25 @@ def test_api_watch_PUT_update(client, live_server):
     # Cleanup everything
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
+
+
+def test_api_import(client, live_server):
+    api_key = extract_api_key_from_UI(client)
+
+    res = client.post(
+        url_for("import") + "?tag=import-test",
+        data='https://website1.com\r\nhttps://website2.com',
+        headers={'x-api-key': api_key},
+        follow_redirects=True
+    )
+
+    assert res.status_code == 200
+    assert len(res.json) == 2
+    res = client.get(url_for("index"))
+    assert b"https://website1.com" in res.data
+    assert b"https://website2.com" in res.data
+
+    # Should see the new tag in the tag/groups list
+    res = client.get(url_for('tags.tags_overview_page'))
+    assert b'import-test' in res.data
+    

changedetectionio/tests/test_notification.py

@@ -281,7 +281,8 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
 
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}"
+    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
 
     res = client.post(
         url_for("settings_page"),
@@ -297,10 +298,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         follow_redirects=True
     )
     assert b'Settings updated' in res.data
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )
+
     # Add a watch and trigger a HTTP POST
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -315,7 +313,9 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     set_modified_response()
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
+    wait_for_all_checks(client)
+
+    time.sleep(2) # plus extra delay for notifications to fire
 
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
@@ -328,6 +328,13 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     with open("test-datastore/notification-url.txt", 'r') as f:
         notification_url = f.read()
         assert 'xxx=http' in notification_url
+        # apprise style headers should be stripped
+        assert 'custom-header' not in notification_url
+
+    with open("test-datastore/notification-headers.txt", 'r') as f:
+        notification_headers = f.read()
+        assert 'custom-header: 123' in notification_headers.lower()
+
 
     # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
     assert os.path.isfile("test-datastore/notification-content-type.txt")
@@ -335,3 +342,8 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         assert 'application/json' in f.read()
 
     os.unlink("test-datastore/notification-url.txt")
+
+    client.get(
+        url_for("form_delete", uuid="all"),
+        follow_redirects=True
+    )

changedetectionio/tests/util.py

@@ -205,6 +205,9 @@ def live_server_setup(live_server):
         with open("test-datastore/notification-url.txt", "w") as f:
             f.write(request.url)
 
+        with open("test-datastore/notification-headers.txt", "w") as f:
+            f.write(str(request.headers))
+
         if request.content_type:
             with open("test-datastore/notification-content-type.txt", "w") as f:
                 f.write(request.content_type)

docker-compose.yml

@@ -90,7 +90,7 @@ services:
 #
 
      # Used for fetching pages via Playwright+Chrome where you need Javascript support.
-     # Note: works well but is deprecated, doesnt fetch full page screenshots and other issues
+     # Note: works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector) and other issues
 #    browser-chrome:
 #        hostname: browser-chrome
 #        image: selenium/standalone-chrome:4

docs/api_v1/index.html

@@ -5,13 +5,13 @@
   <meta name="description" content="Manage your changedetection.io watches via API, requires the `x-api-key` header which is found in the settings UI.">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-  <link href="assets/bootstrap.min.css?v=1677105736053" rel="stylesheet" media="screen">
-  <link href="assets/prism.css?v=1677105736053" rel="stylesheet" />
-  <link href="assets/main.css?v=1677105736053" rel="stylesheet" media="screen, print">
-  <link href="assets/favicon.ico?v=1677105736053" rel="icon" type="image/x-icon">
-  <link href="assets/apple-touch-icon.png?v=1677105736053" rel="apple-touch-icon" sizes="180x180">
-  <link href="assets/favicon-32x32.png?v=1677105736053" rel="icon" type="image/png" sizes="32x32">
-  <link href="assets/favicon-16x16.png?v=1677105736053" rel="icon" type="image/png" sizes="16x16">
+  <link href="assets/bootstrap.min.css?v=1701595483622" rel="stylesheet" media="screen">
+  <link href="assets/prism.css?v=1701595483622" rel="stylesheet" />
+  <link href="assets/main.css?v=1701595483622" rel="stylesheet" media="screen, print">
+  <link href="assets/favicon.ico?v=1701595483622" rel="icon" type="image/x-icon">
+  <link href="assets/apple-touch-icon.png?v=1701595483622" rel="apple-touch-icon" sizes="180x180">
+  <link href="assets/favicon-32x32.png?v=1701595483622" rel="icon" type="image/png" sizes="32x32">
+  <link href="assets/favicon-16x16.png?v=1701595483622" rel="icon" type="image/png" sizes="16x16">
 </head>
 
 <body class="container-fluid">
@@ -928,6 +928,6 @@
   </div>
 </div>
 
-<script src="assets/main.bundle.js?v=1677105736053"></script>
+<script src="assets/main.bundle.js?v=1701595483622"></script>
 </body>
 </html>

setup.py

@@ -32,7 +32,7 @@ setup(
     long_description_content_type='text/markdown',
     keywords='website change monitor for changes notification change detection '
              'alerts tracking website tracker change alert website and monitoring',
-    entry_points={"console_scripts": ["changedetection.io=changedetectionio.changedetection:main"]},
+    entry_points={"console_scripts": ["changedetection.io=changedetectionio:main"]},
     zip_safe=True,
     scripts=["changedetection.py"],
     author='dgtlmoon',