little for apprise type urls that are converted ot post://

.github/workflows/test-only.yml

@@ -98,58 +98,14 @@ jobs:
 
       - name: Test changedetection.io container starts+runs basically without error
         run: |
-          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
+          docker run -p 5556:5000 -d test-changedetectionio
           sleep 3
           # Should return 0 (no error) when grep finds it
           curl -s http://localhost:5556 |grep -q checkbox-uuid
           
           # and IPv6
           curl -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
-          docker kill test-changedetectionio
-
-      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
-        run: |
           
-          echo SIGINT Shutdown request test
-          docker run --name sig-test -d test-changedetectionio
-          sleep 3
-          echo ">>> Sending SIGINT to sig-test container"
-          docker kill --signal=SIGINT sig-test
-          sleep 3
-          # invert the check (it should be not 0/not running)
-          docker ps
-          # check signal catch(STDOUT) log
-          docker logs sig-test | grep 'Shutdown: Got Signal - SIGINT' || exit 1
-          test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]
-          then
-            echo "Looks like container was running when it shouldnt be"
-            docker ps
-            exit 1
-          fi
-          
-          # @todo - scan the container log to see the right "graceful shutdown" text exists 
-          docker rm sig-test
-          
-          echo SIGTERM Shutdown request test
-          docker run --name sig-test -d test-changedetectionio
-          sleep 3
-          echo ">>> Sending SIGTERM to sig-test container"
-          docker kill --signal=SIGTERM sig-test
-          sleep 3
-          # invert the check (it should be not 0/not running)
-          docker ps
-          docker logs sig-test | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
-          test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]
-          then
-            echo "Looks like container was running when it shouldnt be"
-            docker ps
-            exit 1
-          fi
-          
-          # @todo - scan the container log to see the right "graceful shutdown" text exists           
-          docker rm sig-test
 
 #export WEBDRIVER_URL=http://localhost:4444/wd/hub
 #pytest tests/fetchers/test_content.py

Dockerfile

@@ -53,12 +53,12 @@ ENV PYTHONPATH=/usr/local
 
 EXPOSE 5000
 
-# The actual flask app module
+# The actual flask app
 COPY changedetectionio /app/changedetectionio
-# Starting wrapper
+
+# The eventlet server wrapper
 COPY changedetection.py /app/changedetection.py
 
 WORKDIR /app
-CMD ["python", "./changedetection.py", "-d", "/datastore"]
-
 
+CMD [ "python", "./changedetection.py" , "-d", "/datastore"]

README.md

@@ -11,10 +11,9 @@ _Live your data-life pro-actively._
 
 ![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
 
-[**Get started with website page change monitoring straight away. Don't have time? Try our $8.99/month subscription, use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_
+[**Don't have time? Let us host it for you! try our $8.99/month subscription - use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_
 
 - Chrome browser included.
-- Nothing to install, access via browser login after signup.
 - Super fast, no registration needed setup.
 - Get started watching and receiving website change notifications straight away.
 
@@ -98,7 +97,7 @@ Please :star: star :star: this project and help it grow! https://github.com/dgtl
 With Docker composer, just clone this repository and..
 
 ```bash
-$ docker compose up -d
+$ docker-compose up -d
 ```
 
 Docker standalone
@@ -137,10 +136,10 @@ docker rm $(docker ps -a -f name=changedetection.io -q)
 docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
 ```
 
-### docker compose
+### docker-compose
 
 ```bash
-docker compose pull && docker compose up -d
+docker-compose pull && docker-compose up -d
 ```
 
 See the wiki for more information https://github.com/dgtlmoon/changedetection.io/wiki

changedetection.py

@@ -1,6 +1,44 @@
 #!/usr/bin/python3
 
-# Only exists for direct CLI usage
+# Entry-point for running from the CLI when not installed via Pip, Pip will handle the console_scripts entry_points's from setup.py
+# It's recommended to use `pip3 install changedetection.io` and start with `changedetection.py` instead, it will be linkd to your global path.
+# or Docker.
+# Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-import changedetectionio
-changedetectionio.main()
+from changedetectionio import changedetection
+import multiprocessing
+import sys
+import os
+
+def sigchld_handler(_signo, _stack_frame):
+    import sys
+    print('Shutdown: Got SIGCHLD')
+    # https://stackoverflow.com/questions/40453496/python-multiprocessing-capturing-signals-to-restart-child-processes-or-shut-do
+    pid, status = os.waitpid(-1, os.WNOHANG | os.WUNTRACED | os.WCONTINUED)
+
+    print('Sub-process: pid %d status %d' % (pid, status))
+    if status != 0:
+        sys.exit(1)
+
+    raise SystemExit
+
+if __name__ == '__main__':
+
+    #signal.signal(signal.SIGCHLD, sigchld_handler)
+
+    # The only way I could find to get Flask to shutdown, is to wrap it and then rely on the subsystem issuing SIGTERM/SIGKILL
+    parse_process = multiprocessing.Process(target=changedetection.main)
+    parse_process.daemon = True
+    parse_process.start()
+    import time
+
+    try:
+        while True:
+            time.sleep(1)
+            if not parse_process.is_alive():
+                # Process died/crashed for some reason, exit with error set
+                sys.exit(1)
+
+    except KeyboardInterrupt:
+        #parse_process.terminate() not needed, because this process will issue it to the sub-process anyway
+        print ("Exited - CTRL+C")

changedetectionio/api/api_v1.py

@@ -304,7 +304,7 @@ class Import(Resource):
     @auth.check_token
     def post(self):
         """
-        @api {post} /api/v1/import Import a list of watched URLs
+        @api {post} /api/v1/import - Import a list of watched URLs
         @apiDescription Accepts a line-feed separated list of URLs to import, additionally with ?tag_uuids=(tag  id), ?tag=(name), ?proxy={key}, ?dedupe=true (default true) one URL per line.
         @apiExample {curl} Example usage:
             curl http://localhost:5000/api/v1/import --data-binary @list-of-sites.txt -H"x-api-key:8a111a21bc2f8f1dd9b9353bbd46049a"
@@ -351,6 +351,7 @@ class Import(Resource):
 
         return added
 
+
 class SystemInfo(Resource):
     def __init__(self, **kwargs):
         # datastore is a black box dependency

changedetectionio/blueprint/browser_steps/__init__.py

@@ -25,9 +25,10 @@ from distutils.util import strtobool
 from flask import Blueprint, request, make_response
 import logging
 import os
+import re
 
 from changedetectionio.store import ChangeDetectionStore
-from changedetectionio.flask_app import login_optionally_required
+from changedetectionio import login_optionally_required
 
 browsersteps_sessions = {}
 io_interface_context = None

changedetectionio/blueprint/tags/__init__.py

@@ -1,6 +1,6 @@
 from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
 from changedetectionio.store import ChangeDetectionStore
-from changedetectionio.flask_app import login_optionally_required
+from changedetectionio import login_optionally_required
 
 
 def construct_blueprint(datastore: ChangeDetectionStore):

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -3,7 +3,7 @@
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>

changedetectionio/changedetection.py

@@ -0,0 +1,153 @@
+#!/usr/bin/python3
+
+# Launch as a eventlet.wsgi server instance.
+
+from distutils.util import strtobool
+from json.decoder import JSONDecodeError
+
+import eventlet
+import eventlet.wsgi
+import getopt
+import os
+import signal
+import socket
+import sys
+
+from . import store, changedetection_app, content_fetcher
+from . import __version__
+
+# Only global so we can access it in the signal handler
+app = None
+datastore = None
+
+def sigterm_handler(_signo, _stack_frame):
+    global app
+    global datastore
+#    app.config.exit.set()
+    print('Shutdown: Got SIGTERM, DB saved to disk')
+    datastore.sync_to_json()
+#    raise SystemExit
+
+def main():
+    global datastore
+    global app
+
+    datastore_path = None
+    do_cleanup = False
+    host = ''
+    ipv6_enabled = False
+    port = os.environ.get('PORT') or 5000
+    ssl_mode = False
+
+    # On Windows, create and use a default path.
+    if os.name == 'nt':
+        datastore_path = os.path.expandvars(r'%APPDATA%\changedetection.io')
+        os.makedirs(datastore_path, exist_ok=True)
+    else:
+        # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
+        datastore_path = os.path.join(os.getcwd(), "../datastore")
+
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:", "port")
+    except getopt.GetoptError:
+        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
+        sys.exit(2)
+
+    create_datastore_dir = False
+
+    for opt, arg in opts:
+        if opt == '-s':
+            ssl_mode = True
+
+        if opt == '-h':
+            host = arg
+
+        if opt == '-p':
+            port = int(arg)
+
+        if opt == '-d':
+            datastore_path = arg
+
+        if opt == '-6':
+            print ("Enabling IPv6 listen support")
+            ipv6_enabled = True
+
+        # Cleanup (remove text files that arent in the index)
+        if opt == '-c':
+            do_cleanup = True
+
+        # Create the datadir if it doesnt exist
+        if opt == '-C':
+            create_datastore_dir = True
+
+    # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
+    app_config = {'datastore_path': datastore_path}
+
+    if not os.path.isdir(app_config['datastore_path']):
+        if create_datastore_dir:
+            os.mkdir(app_config['datastore_path'])
+        else:
+            print(
+                "ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists or specify a directory with the -d option.\n"
+                "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
+            sys.exit(2)
+
+    try:
+        datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
+    except JSONDecodeError as e:
+        # Dont' start if the JSON DB looks corrupt
+        print ("ERROR: JSON DB or Proxy List JSON at '{}' appears to be corrupt, aborting".format(app_config['datastore_path']))
+        print(str(e))
+        return
+
+    app = changedetection_app(app_config, datastore)
+
+    signal.signal(signal.SIGTERM, sigterm_handler)
+
+    # Go into cleanup mode
+    if do_cleanup:
+        datastore.remove_unused_snapshots()
+
+    app.config['datastore_path'] = datastore_path
+
+
+    @app.context_processor
+    def inject_version():
+        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
+                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
+                    has_password=datastore.data['settings']['application']['password'] != False
+                    )
+
+    # Monitored websites will not receive a Referer header when a user clicks on an outgoing link.
+    # @Note: Incompatible with password login (and maybe other features) for now, submit a PR!
+    @app.after_request
+    def hide_referrer(response):
+        if strtobool(os.getenv("HIDE_REFERER", 'false')):
+            response.headers["Referrer-Policy"] = "no-referrer"
+
+        return response
+
+    # Proxy sub-directory support
+    # Set environment var USE_X_SETTINGS=1 on this script
+    # And then in your proxy_pass settings
+    #
+    #         proxy_set_header Host "localhost";
+    #         proxy_set_header X-Forwarded-Prefix /app;
+
+    if os.getenv('USE_X_SETTINGS'):
+        print ("USE_X_SETTINGS is ENABLED\n")
+        from werkzeug.middleware.proxy_fix import ProxyFix
+        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
+
+    s_type = socket.AF_INET6 if ipv6_enabled else socket.AF_INET
+
+    if ssl_mode:
+        # @todo finalise SSL config, but this should get you in the right direction if you need it.
+        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port), s_type),
+                                               certfile='cert.pem',
+                                               keyfile='privkey.pem',
+                                               server_side=True), app)
+
+    else:
+        eventlet.wsgi.server(eventlet.listen((host, int(port)), s_type), app)
+

changedetectionio/content_fetcher.py

@@ -43,11 +43,9 @@ class JSActionExceptions(Exception):
         return
 
 
-class BrowserStepsStepException(Exception):
-    def __init__(self, step_n, original_e):
+class BrowserStepsStepTimout(Exception):
+    def __init__(self, step_n):
         self.step_n = step_n
-        self.original_e = original_e
-        print(f"Browser Steps exception at step {self.step_n}", str(original_e))
         return
 
 
@@ -93,20 +91,19 @@ class ReplyWithContentButNoText(Exception):
 
 
 class Fetcher():
-    browser_connection_is_custom = None
-    browser_connection_url = None
     browser_steps = None
     browser_steps_screenshot_path = None
     content = None
     error = None
     fetcher_description = "No description"
+    browser_connection_url = None
     headers = {}
-    instock_data = None
-    instock_data_js = ""
     status_code = None
     webdriver_js_execute_code = None
     xpath_data = None
     xpath_element_js = ""
+    instock_data = None
+    instock_data_js = ""
 
     # Will be needed in the future by the VisualSelector, always get this where possible.
     screenshot = False
@@ -175,7 +172,7 @@ class Fetcher():
 
     def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-        from playwright._impl._errors import TimeoutError, Error
+        from playwright._impl._errors import TimeoutError
         from jinja2 import Environment
         jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
 
@@ -205,10 +202,10 @@ class Fetcher():
                                                       optional_value=optional_value)
                     self.screenshot_step(step_n)
                     self.save_step_html(step_n)
-
-                except (Error, TimeoutError) as e:
+                except TimeoutError as e:
+                    print(str(e))
                     # Stop processing here
-                    raise BrowserStepsStepException(step_n=step_n, original_e=e)
+                    raise BrowserStepsStepTimout(step_n=step_n)
 
     # It's always good to reset these
     def delete_browser_steps_screenshots(self):
@@ -255,19 +252,16 @@ class base_html_playwright(Fetcher):
 
     proxy = None
 
-    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+    def __init__(self, proxy_override=None, browser_connection_url=None):
         super().__init__()
 
         self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
 
-        if custom_browser_connection_url:
-            self.browser_connection_is_custom = True
-            self.browser_connection_url = custom_browser_connection_url
-        else:
-            # Fallback to fetching from system
-            # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+        if not browser_connection_url:
             self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
-
+        else:
+            self.browser_connection_url = browser_connection_url
 
         # If any proxy settings are enabled, then we should setup the proxy object
         proxy_args = {}
@@ -427,10 +421,8 @@ class base_html_playwright(Fetcher):
             current_include_filters=None,
             is_binary=False):
 
-
         # For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!)
-        # browser_connection_is_custom doesnt work with puppeteer style fetch (use playwright native too in this case)
-        if not self.browser_connection_is_custom and not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
+        if not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
             if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')):
                 # Temporary backup solution until we rewrite the playwright code
                 return self.run_fetch_browserless_puppeteer(
@@ -577,16 +569,15 @@ class base_html_webdriver(Fetcher):
                                         'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
     proxy = None
 
-    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+    def __init__(self, proxy_override=None, browser_connection_url=None):
         super().__init__()
         from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
 
         # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-        if not custom_browser_connection_url:
+        if not browser_connection_url:
             self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
         else:
-            self.browser_connection_is_custom = True
-            self.browser_connection_url = custom_browser_connection_url
+            self.browser_connection_url = browser_connection_url
 
         # If any proxy settings are enabled, then we should setup the proxy object
         proxy_args = {}
@@ -683,7 +674,7 @@ class base_html_webdriver(Fetcher):
 class html_requests(Fetcher):
     fetcher_description = "Basic fast Plaintext/HTTP Client"
 
-    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+    def __init__(self, proxy_override=None, browser_connection_url=None):
         super().__init__()
         self.proxy_override = proxy_override
         # browser_connection_url is none because its always 'launched locally'

changedetectionio/model/Watch.py

@@ -38,7 +38,6 @@ base_config = {
     'track_ldjson_price_data': None,
     'headers': {},  # Extra headers to send
     'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
-    'in_stock' : None,
     'in_stock_only' : True, # Only trigger change on going to instock from out-of-stock
     'include_filters': [],
     'last_checked': 0,

changedetectionio/notification.py

@@ -90,6 +90,7 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
             if not k.strip('+-') in results['qsd+'].keys():
                 params[URLBase.unquote(k)] = URLBase.unquote(v)
 
+
         # Determine Authentication
         auth = ''
         if results.get('user') and results.get('password'):
@@ -105,10 +106,10 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
         pass
 
     r(results.get('url'),
-      auth=auth,
-      data=body,
       headers=headers,
-      params=params
+      data=body,
+      params=params,
+      auth=auth
       )
 
 
@@ -221,14 +222,13 @@ def process_notification(n_object, datastore):
 
 
 # Notification title + body content parameters get created here.
-# ( Where we prepare the tokens in the notification to be replaced with actual values )
 def create_notification_parameters(n_object, datastore):
     from copy import deepcopy
 
     # in the case we send a test notification from the main settings, there is no UUID.
     uuid = n_object['uuid'] if 'uuid' in n_object else ''
 
-    if uuid:
+    if uuid != '':
         watch_title = datastore.data['watching'][uuid].get('title', '')
         tag_list = []
         tags = datastore.get_all_tags_for_watch(uuid)
@@ -256,7 +256,7 @@ def create_notification_parameters(n_object, datastore):
     tokens.update(
         {
             'base_url': base_url,
-            'current_snapshot': n_object.get('current_snapshot', ''),
+            'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '',
             'diff': n_object.get('diff', ''),  # Null default in the case we use a test
             'diff_added': n_object.get('diff_added', ''),  # Null default in the case we use a test
             'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test

changedetectionio/processors/__init__.py

@@ -43,20 +43,15 @@ class difference_detection_processor():
 
         # In the case that the preferred fetcher was a browser config with custom connection URL..
         # @todo - on save watch, if its extra_browser_ then it should be obvious it will use playwright (like if its requests now..)
-        custom_browser_connection_url = None
+        browser_connection_url = None
         if prefer_fetch_backend.startswith('extra_browser_'):
             (t, key) = prefer_fetch_backend.split('extra_browser_')
             connection = list(
                 filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', [])))
             if connection:
                 prefer_fetch_backend = 'base_html_playwright'
-                custom_browser_connection_url = connection[0].get('browser_connection_url')
+                browser_connection_url = connection[0].get('browser_connection_url')
 
-        # PDF should be html_requests because playwright will serve it up (so far) in a embedded page
-        # @todo https://github.com/dgtlmoon/changedetection.io/issues/2019
-        # @todo needs test to or a fix
-        if self.watch.is_pdf:
-           prefer_fetch_backend = "html_requests"
 
         # Grab the right kind of 'fetcher', (playwright, requests, etc)
         if hasattr(content_fetcher, prefer_fetch_backend):
@@ -74,7 +69,7 @@ class difference_detection_processor():
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
         self.fetcher = fetcher_obj(proxy_override=proxy_url,
-                                   custom_browser_connection_url=custom_browser_connection_url
+                                   browser_connection_url=browser_connection_url
                                    )
 
         if self.watch.has_browser_steps:

changedetectionio/res/stock-not-in-stock.js

@@ -4,7 +4,6 @@ function isItemInStock() {
     ' أخبرني عندما يتوفر',
     '0 in stock',
     'agotado',
-    'article épuisé',
     'artikel zurzeit vergriffen',
     'as soon as stock is available',
     'ausverkauft', // sold out
@@ -18,8 +17,10 @@ function isItemInStock() {
     'currently have any tickets for this',
     'currently unavailable',
     'dostępne wkrótce',
+    'dostępne wkrótce',
     'en rupture de stock',
     'ist derzeit nicht auf lager',
+    'ist derzeit nicht auf lager',
     'item is no longer available',
     'let me know when it\'s available',
     'message if back in stock',
@@ -27,8 +28,6 @@ function isItemInStock() {
     'nicht auf lager',
     'nicht lieferbar',
     'nicht zur verfügung',
-    'niet beschikbaar',
-    'niet leverbaar',
     'no disponible temporalmente',
     'no longer in stock',
     'no tickets available',
@@ -41,11 +40,9 @@ function isItemInStock() {
     'out-of-stock',
     'produkt niedostępny',
     'sold out',
-    'sold-out',
     'temporarily out of stock',
     'temporarily unavailable',
     'tickets unavailable',
-    'tijdelijk uitverkocht',
     'unavailable tickets',
     'we do not currently have an estimate of when this product will be back in stock.',
     'zur zeit nicht an lager',
@@ -114,4 +111,4 @@ function isItemInStock() {
 }
 
 // returns the element text that makes it think it's out of stock
-return isItemInStock();
+return isItemInStock();

changedetectionio/static/js/notifications.js

@@ -24,17 +24,14 @@ $(document).ready(function() {
     })
 
     data = {
-      notification_body: $('#notification_body').val(),
-      notification_format: $('#notification_format').val(),
-      notification_title: $('#notification_title').val(),
-      notification_urls: $('.notification-urls').val(),
-      window_url: window.location.href,
+        window_url : window.location.href,
+        notification_urls : $('.notification-urls').val(),
     }
-
-
-    if (!data['notification_urls'].length) {
-      alert("Notification URL list is empty, cannot send test.")
-      return;
+    for (key in data) {
+      if (!data[key].length) {
+        alert(key+" is empty, cannot send test.")
+        return;
+      }
     }
 
     $.ajax({

changedetectionio/static/js/vis.js

@@ -2,28 +2,21 @@ $(document).ready(function () {
 
     // Lazy Hide/Show elements mechanism
     $('[data-visible-for]').hide();
-    function show_related_elem(e) {
-        var n = $(e).attr('name') + "=" + $(e).val();
+    $(':radio').on('keyup keypress blur change click', function (e) {
+        $('[data-visible-for]').hide();
+        $('.advanced-options').hide();
+        var n = $(this).attr('name') + "=" + $(this).val();
         if (n === 'fetch_backend=system') {
             n = "fetch_backend=" + default_system_fetch_backend;
         }
         $(`[data-visible-for~="${n}"]`).show();
-    }
-    $(':radio').on('keyup keypress blur change click', function (e) {
-        $(`[data-visible-for]`).hide();
-        $('.advanced-options').hide();
-        show_related_elem(this);
-    });
 
-    $(':radio:checked').each(function (e) {
-       show_related_elem(this);
-    })
+    });
+    $(':radio:checked').change();
 
 
     // Show advanced
     $('.show-advanced').click(function (e) {
-        $(this).closest('.tab-pane-inner').find('.advanced-options').each(function (e) {
-            $(this).toggle();
-        })
+        $(this).closest('.tab-pane-inner').find('.advanced-options').toggle();
     });
 });

changedetectionio/static/js/watch-settings.js

@@ -1,4 +1,18 @@
 $(document).ready(function () {
+
+    // Lazy Hide/Show elements mechanism
+    $('[data-visible-for]').hide();
+    $(':radio').on('keyup keypress blur change click', function (e){
+        $('[data-visible-for]').hide();
+        var n = $(this).attr('name') + "=" + $(this).val();
+        if (n === 'fetch_backend=system') {
+            n = "fetch_backend=" + default_system_fetch_backend;
+        }
+        $(`[data-visible-for~="${n}"]`).show();
+
+    });
+    $(':radio:checked').change();
+
     $('#notification-setting-reset-to-default').click(function (e) {
         $('#notification_title').val('');
         $('#notification_body').val('');

changedetectionio/store.py

@@ -248,7 +248,6 @@ class ChangeDetectionStore:
                 'check_count': 0,
                 'fetch_time' : 0.0,
                 'has_ldjson_price_data': None,
-                'in_stock': None,
                 'last_checked': 0,
                 'last_error': False,
                 'last_notification_error': False,

changedetectionio/templates/_common_fields.jinja

@@ -16,7 +16,7 @@
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
-                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>) <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes#postposts">more help here</a></li>
+                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
                                   <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                               </ul>
                             </div>

changedetectionio/templates/edit.html

@@ -14,7 +14,7 @@
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
 {% endif %}
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
     const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
     const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
     const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
@@ -127,7 +127,7 @@
                 {% endif %}
 
                 <!-- webdriver always -->
-                <fieldset data-visible-for="fetch_backend=html_webdriver"  style="display: none;">
+                <fieldset data-visible-for="fetch_backend=html_webdriver">
                     <div class="pure-control-group">
                         {{ render_field(form.webdriver_delay) }}
                         <div class="pure-form-message-inline">
@@ -153,15 +153,15 @@
                     </div>
                 </fieldset>
                 <!-- html requests always -->
-                <fieldset data-visible-for="fetch_backend=html_requests">
+                <fieldset data-visible-for="fetch_backend=html_requests" style="display: none;">
                     <div class="pure-control-group">
                         <a class="pure-button button-secondary button-xsmall show-advanced">Show advanced options</a>
                     </div>
                     <div class="advanced-options"  style="display: none;">
-                        <div class="pure-control-group" id="request-method">
+                        <div class="pure-control-group advanced-options" id="request-method" style="display: none;">
                             {{ render_field(form.method) }}
                         </div>
-                        <div id="request-body">
+                        <div class="advanced-options" id="request-body">
                                             {{ render_field(form.body, rows=5, placeholder="Example
 {
    \"name\":\"John\",
@@ -187,8 +187,8 @@ User-Agent: wonderbra 1.0") }}
                             (Not supported by Selenium browser)
                         </div>
                     </div>
-            <fieldset data-visible-for="fetch_backend=html_requests fetch_backend=html_webdriver" >
-                    <div class="pure-control-group inline-radio advanced-options"  style="display: none;">
+            <fieldset data-visible-for="fetch_backend=html_requests fetch_backend=html_webdriver" style="display: none;">
+                    <div  class="pure-control-group inline-radio advanced-options"  style="display: none;">
                     {{ render_checkbox_field(form.ignore_status_codes) }}
                     </div>
             </fieldset>

changedetectionio/templates/settings.html

@@ -4,7 +4,7 @@
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
 {% endif %}
@@ -236,11 +236,8 @@ nav
                 <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
                 </div>
                 <div class="pure-control-group" id="extra-browsers-setting">
-                    <p>
-                    <span class="pure-form-message-inline"><i>Extra Browsers</i> can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.</span><br>
-                    <span class="pure-form-message-inline">Simply paste the connection address into the box, <a href="https://changedetection.io/tutorial/using-bright-datas-scraping-browser-pass-captchas-and-other-protection-when-monitoring">More instructions and examples here</a> </span>
-                    </p>
-                    {{ render_field(form.requests.form.extra_browsers) }}
+                    <span class="pure-form-message-inline"><i>Extra Browsers</i> allow changedetection.io to communicate with a different web-browser.</span><br>
+                  {{ render_field(form.requests.form.extra_browsers) }}
                 </div>
             </div>
             <div id="actions">

changedetectionio/templates/watch-overview.html

@@ -141,7 +141,7 @@
                     {% if watch['processor'] == 'restock_diff'  %}
                     <span class="restock-label {{'in-stock' if watch['in_stock'] else 'not-in-stock' }}" title="detecting restock conditions">
                         <!-- maybe some object watch['processor'][restock_diff] or.. -->
-                        {% if watch['last_checked'] and watch['in_stock'] != None %}
+                        {% if watch['last_checked'] %}
                             {% if watch['in_stock'] %} In stock {% else %} Not in stock {% endif %}
                         {% else %}
                             Not yet checked

changedetectionio/tests/conftest.py

@@ -13,17 +13,22 @@ global app
 
 
 def cleanup(datastore_path):
-    import glob
     # Unlink test output files
-
-    for g in ["*.txt", "*.json", "*.pdf"]:
-        files = glob.glob(os.path.join(datastore_path, g))
-        for f in files:
-            if 'proxies.json' in f:
-                # Usually mounted by docker container during test time
-                continue
-            if os.path.isfile(f):
-                os.unlink(f)
+    files = [
+        'count.txt',
+        'endpoint-content.txt'
+        'headers.txt',
+        'headers-testtag.txt',
+        'notification.txt',
+        'secret.txt',
+        'url-watches.json',
+        'output.txt',
+    ]
+    for file in files:
+        try:
+            os.unlink("{}/{}".format(datastore_path, file))
+        except FileNotFoundError:
+            pass
 
 @pytest.fixture(scope='session')
 def app(request):

changedetectionio/tests/test_api.py

@@ -378,4 +378,3 @@ def test_api_import(client, live_server):
     # Should see the new tag in the tag/groups list
     res = client.get(url_for('tags.tags_overview_page'))
     assert b'import-test' in res.data
-    

changedetectionio/tests/test_notification.py

@@ -281,8 +281,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
 
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
-    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}"
 
     res = client.post(
         url_for("settings_page"),
@@ -298,7 +297,10 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         follow_redirects=True
     )
     assert b'Settings updated' in res.data
-
+    client.get(
+        url_for("form_delete", uuid="all"),
+        follow_redirects=True
+    )
     # Add a watch and trigger a HTTP POST
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -313,9 +315,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     set_modified_response()
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(2) # plus extra delay for notifications to fire
+    time.sleep(2)
 
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
@@ -328,13 +328,6 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     with open("test-datastore/notification-url.txt", 'r') as f:
         notification_url = f.read()
         assert 'xxx=http' in notification_url
-        # apprise style headers should be stripped
-        assert 'custom-header' not in notification_url
-
-    with open("test-datastore/notification-headers.txt", 'r') as f:
-        notification_headers = f.read()
-        assert 'custom-header: 123' in notification_headers.lower()
-
 
     # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
     assert os.path.isfile("test-datastore/notification-content-type.txt")
@@ -342,8 +335,3 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         assert 'application/json' in f.read()
 
     os.unlink("test-datastore/notification-url.txt")
-
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )

changedetectionio/tests/util.py

@@ -205,9 +205,6 @@ def live_server_setup(live_server):
         with open("test-datastore/notification-url.txt", "w") as f:
             f.write(request.url)
 
-        with open("test-datastore/notification-headers.txt", "w") as f:
-            f.write(str(request.headers))
-
         if request.content_type:
             with open("test-datastore/notification-content-type.txt", "w") as f:
                 f.write(request.content_type)

changedetectionio/update_worker.py

@@ -26,61 +26,47 @@ class update_worker(threading.Thread):
         self.datastore = datastore
         super().__init__(*args, **kwargs)
 
-    def queue_notification_for_watch(self, notification_q, n_object, watch):
+    def queue_notification_for_watch(self, n_object, watch):
 
         from changedetectionio import diff
-        dates = []
-        trigger_text = ''
-
-        if watch:
-            watch_history = watch.history
-            dates = list(watch_history.keys())
-            trigger_text = watch.get('trigger_text', [])
 
+        watch_history = watch.history
+        dates = list(watch_history.keys())
         # Add text that was triggered
-        if len(dates):
-            snapshot_contents = watch.get_history_snapshot(dates[-1])
-        else:
-            snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
+        snapshot_contents = watch.get_history_snapshot(dates[-1])
 
         # HTML needs linebreak, but MarkDown and Text can use a linefeed
-        if n_object.get('notification_format') == 'HTML':
+        if n_object['notification_format'] == 'HTML':
             line_feed_sep = "<br>"
             # Snapshot will be plaintext on the disk, convert to some kind of HTML
             snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
         else:
             line_feed_sep = "\n"
 
+        trigger_text = watch.get('trigger_text', [])
         triggered_text = ''
+
         if len(trigger_text):
             from . import html_tools
             triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
             if triggered_text:
                 triggered_text = line_feed_sep.join(triggered_text)
 
-        # Could be called as a 'test notification' with only 1 snapshot available
-        prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
-        current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
-
-        if len(dates) > 1:
-            prev_snapshot = watch.get_history_snapshot(dates[-2])
-            current_snapshot = watch.get_history_snapshot(dates[-1])
 
         n_object.update({
             'current_snapshot': snapshot_contents,
-            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep),
-            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
-            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep),
-            'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
-            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
-            'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
+            'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
+            'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
+            'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
+            'diff_patch': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep, patch_format=True),
+            'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
+            'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
             'triggered_text': triggered_text,
-            'uuid': watch.get('uuid') if watch else None,
-            'watch_url': watch.get('url') if watch else None,
+            'uuid': watch.get('uuid'),
+            'watch_url': watch.get('url'),
         })
-        logging.info(">> SENDING NOTIFICATION")
-
-        notification_q.put(n_object)
+        logging.info (">> SENDING NOTIFICATION")
+        self.notification_q.put(n_object)
 
     # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
     def _check_cascading_vars(self, var_name, watch):
@@ -148,7 +134,7 @@ class update_worker(threading.Thread):
         queued = False
         if n_object and n_object.get('notification_urls'):
             queued = True
-            self.queue_notification_for_watch(notification_q=self.notification_q, n_object=n_object, watch=watch)
+            self.queue_notification_for_watch(n_object, watch)
 
         return queued
 
@@ -188,9 +174,9 @@ class update_worker(threading.Thread):
             return
         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
         n_object = {'notification_title': "Changedetection.io - Alert - Browser step at position {} could not be run".format(step_n+1),
-                    'notification_body': "Your configured browser step at position {} for {{{{watch_url}}}} "
+                    'notification_body': "Your configured browser step at position {} for {{watch['url']}} "
                                          "did not appear on the page after {} attempts, did the page change layout? "
-                                         "Does it need a delay added?\n\nLink: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\n"
+                                         "Does it need a delay added?\n\nLink: {{base_url}}/edit/{{watch_uuid}}\n\n"
                                          "Thanks - Your omniscient changedetection.io installation :)\n".format(step_n+1, threshold),
                     'notification_format': 'text'}
 
@@ -354,32 +340,20 @@ class update_worker(threading.Thread):
                         changed_detected = False
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False})
 
-                    except content_fetcher.BrowserStepsStepException as e:
+                    except content_fetcher.BrowserStepsStepTimout as e:
 
                         if not self.datastore.data['watching'].get(uuid):
                             continue
 
                         error_step = e.step_n + 1
-                        from playwright._impl._errors import TimeoutError, Error
-
-                        # Generally enough info for TimeoutError (couldnt locate the element after default seconds)
-                        err_text = f"Browser step at position {error_step} could not run, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step."
-
-                        if e.original_e.name == "TimeoutError":
-                            # Just the first line is enough, the rest is the stack trace
-                            err_text += " Could not find the target."
-                        else:
-                            # Other Error, more info is good.
-                            err_text += " " + str(e.original_e).splitlines()[0]
-
-                        print(f"BrowserSteps exception at step {error_step}", str(e.original_e))
-
+                        err_text = f"Warning, browser step at position {error_step} could not run, target not found, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step"
                         self.datastore.update_watch(uuid=uuid,
                                                     update_obj={'last_error': err_text,
                                                                 'browser_steps_last_error_step': error_step
                                                                 }
                                                     )
 
+
                         if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
                             c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
                             c += 1

docker-compose.yml

@@ -90,7 +90,7 @@ services:
 #
 
      # Used for fetching pages via Playwright+Chrome where you need Javascript support.
-     # Note: works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector) and other issues
+     # Note: works well but is deprecated, doesnt fetch full page screenshots and other issues
 #    browser-chrome:
 #        hostname: browser-chrome
 #        image: selenium/standalone-chrome:4

docs/api_v1/index.html

@@ -5,13 +5,13 @@
   <meta name="description" content="Manage your changedetection.io watches via API, requires the `x-api-key` header which is found in the settings UI.">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-  <link href="assets/bootstrap.min.css?v=1701595483622" rel="stylesheet" media="screen">
-  <link href="assets/prism.css?v=1701595483622" rel="stylesheet" />
-  <link href="assets/main.css?v=1701595483622" rel="stylesheet" media="screen, print">
-  <link href="assets/favicon.ico?v=1701595483622" rel="icon" type="image/x-icon">
-  <link href="assets/apple-touch-icon.png?v=1701595483622" rel="apple-touch-icon" sizes="180x180">
-  <link href="assets/favicon-32x32.png?v=1701595483622" rel="icon" type="image/png" sizes="32x32">
-  <link href="assets/favicon-16x16.png?v=1701595483622" rel="icon" type="image/png" sizes="16x16">
+  <link href="assets/bootstrap.min.css?v=1677105736053" rel="stylesheet" media="screen">
+  <link href="assets/prism.css?v=1677105736053" rel="stylesheet" />
+  <link href="assets/main.css?v=1677105736053" rel="stylesheet" media="screen, print">
+  <link href="assets/favicon.ico?v=1677105736053" rel="icon" type="image/x-icon">
+  <link href="assets/apple-touch-icon.png?v=1677105736053" rel="apple-touch-icon" sizes="180x180">
+  <link href="assets/favicon-32x32.png?v=1677105736053" rel="icon" type="image/png" sizes="32x32">
+  <link href="assets/favicon-16x16.png?v=1677105736053" rel="icon" type="image/png" sizes="16x16">
 </head>
 
 <body class="container-fluid">
@@ -928,6 +928,6 @@
   </div>
 </div>
 
-<script src="assets/main.bundle.js?v=1701595483622"></script>
+<script src="assets/main.bundle.js?v=1677105736053"></script>
 </body>
 </html>

requirements.txt

@@ -30,7 +30,7 @@ dnspython~=2.4 # related to eventlet fixes
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise~=1.7.1
+apprise~=1.6.0
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt

setup.py

@@ -32,7 +32,7 @@ setup(
     long_description_content_type='text/markdown',
     keywords='website change monitor for changes notification change detection '
              'alerts tracking website tracker change alert website and monitoring',
-    entry_points={"console_scripts": ["changedetection.io=changedetectionio:main"]},
+    entry_points={"console_scripts": ["changedetection.io=changedetectionio.changedetection:main"]},
     zip_safe=True,
     scripts=["changedetection.py"],
     author='dgtlmoon',