second attempt at plugins

2026-04-28 22:07:06 +00:00 · 2023-11-25 20:05:18 +01:00
62 changed files with 2274 additions and 2573 deletions
@@ -1,72 +0,0 @@
-name: Publish Python 🐍distribution 📦 to PyPI and TestPyPI
-
-on: push
-jobs:
-  build:
-    name: Build distribution 📦
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.x"
-    - name: Install pypa/build
-      run: >-
-        python3 -m
-        pip install
-        build
-        --user
-    - name: Build a binary wheel and a source tarball
-      run: python3 -m build
-    - name: Store the distribution packages
-      uses: actions/upload-artifact@v3
-      with:
-        name: python-package-distributions
-        path: dist/
-
-
-  test-pypi-package:
-    name: Test the built 📦 package works basically.
-    runs-on: ubuntu-latest
-    needs:
-    - build
-    steps:
-    - name: Download all the dists
-      uses: actions/download-artifact@v3
-      with:
-        name: python-package-distributions
-        path: dist/
-    - name: Test that the basic pip built package runs without error
-      run: |
-        set -e
-        pip3 install dist/changedetection.io*.whl
-        changedetection.io -d /tmp -p 10000 &
-        sleep 3
-        curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
-        curl http://127.0.0.1:10000/ >/dev/null
-        killall changedetection.io
-
-
-  publish-to-pypi:
-    name: >-
-      Publish Python 🐍 distribution 📦 to PyPI
-    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
-    needs:
-    - test-pypi-package
-    runs-on: ubuntu-latest
-    environment:
-      name: release
-      url: https://pypi.org/p/changedetection.io
-    permissions:
-      id-token: write  # IMPORTANT: mandatory for trusted publishing
-
-    steps:
-    - name: Download all the dists
-      uses: actions/download-artifact@v3
-      with:
-        name: python-package-distributions
-        path: dist/
-    - name: Publish distribution 📦 to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
@@ -98,58 +98,14 @@ jobs:

      - name: Test changedetection.io container starts+runs basically without error
        run: |
-          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
+          docker run -p 5556:5000 -d test-changedetectionio
          sleep 3
          # Should return 0 (no error) when grep finds it
          curl -s http://localhost:5556 |grep -q checkbox-uuid
          
          # and IPv6
          curl -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
-          docker kill test-changedetectionio
-
-      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
-        run: |
          
-          echo SIGINT Shutdown request test
-          docker run --name sig-test -d test-changedetectionio
-          sleep 3
-          echo ">>> Sending SIGINT to sig-test container"
-          docker kill --signal=SIGINT sig-test
-          sleep 3
-          # invert the check (it should be not 0/not running)
-          docker ps
-          # check signal catch(STDOUT) log
-          docker logs sig-test | grep 'Shutdown: Got Signal - SIGINT' || exit 1
-          test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]
-          then
-            echo "Looks like container was running when it shouldnt be"
-            docker ps
-            exit 1
-          fi
-          
-          # @todo - scan the container log to see the right "graceful shutdown" text exists 
-          docker rm sig-test
-          
-          echo SIGTERM Shutdown request test
-          docker run --name sig-test -d test-changedetectionio
-          sleep 3
-          echo ">>> Sending SIGTERM to sig-test container"
-          docker kill --signal=SIGTERM sig-test
-          sleep 3
-          # invert the check (it should be not 0/not running)
-          docker ps
-          docker logs sig-test | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
-          test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]
-          then
-            echo "Looks like container was running when it shouldnt be"
-            docker ps
-            exit 1
-          fi
-          
-          # @todo - scan the container log to see the right "graceful shutdown" text exists           
-          docker rm sig-test

 #export WEBDRIVER_URL=http://localhost:4444/wd/hub
 #pytest tests/fetchers/test_content.py
@@ -0,0 +1,36 @@
+name: ChangeDetection.io PIP package test
+
+# Triggers the workflow on push or pull request events
+
+# This line doesnt work, even tho it is the documented one
+on: [push, pull_request]
+
+  # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
+  # @todo: some kind of path filter for requirements.txt and Dockerfile
+jobs:
+  test-pip-build-basics:
+    runs-on: ubuntu-latest
+    steps:
+        - uses: actions/checkout@v4
+
+        - name: Set up Python 3.11
+          uses: actions/setup-python@v4
+          with:
+            python-version: 3.11
+
+
+        - name: Test that the basic pip built package runs without error
+          run: |
+            set -e
+            mkdir dist
+            pip3 install wheel
+            python3 setup.py bdist_wheel            
+            pip3 install -r requirements.txt
+            rm ./changedetection.py
+            rm -rf changedetectio
+            
+            pip3 install dist/changedetection.io*.whl
+            changedetection.io -d /tmp -p 10000 &
+            sleep 3
+            curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
+            killall -9 changedetection.io
@@ -25,7 +25,7 @@ RUN pip install --target=/dependencies -r /requirements.txt
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
 # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
-RUN pip install --target=/dependencies playwright~=1.40 \
+RUN pip install --target=/dependencies playwright~=1.39 \
    || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."

 # Final image stage
@@ -53,12 +53,12 @@ ENV PYTHONPATH=/usr/local

 EXPOSE 5000

-# The actual flask app module
+# The actual flask app
 COPY changedetectionio /app/changedetectionio
-# Starting wrapper
+
+# The eventlet server wrapper
 COPY changedetection.py /app/changedetection.py

 WORKDIR /app
-CMD ["python", "./changedetection.py", "-d", "/datastore"]
-

+CMD [ "python", "./changedetection.py" , "-d", "/datastore"]
@@ -10,8 +10,6 @@ prune changedetectionio/static/package-lock.json
 prune changedetectionio/static/styles/node_modules
 prune changedetectionio/static/styles/package-lock.json
 include changedetection.py
-include requirements.txt
-include README-pip.md
 global-exclude *.pyc
 global-exclude node_modules
 global-exclude venv
@@ -0,0 +1 @@
+web: python3 ./changedetection.py -C -d ./datastore -p $PORT
@@ -11,13 +11,12 @@ _Live your data-life pro-actively._

 ![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)

-[**Get started with website page change monitoring straight away. Don't have time? Try our $8.99/month subscription, use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_
+[**Don't have time? Let us host it for you! try our $8.99/month subscription - use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_

 - Chrome browser included.
- Nothing to install, access via browser login after signup.
 - Super fast, no registration needed setup.
 - Get started watching and receiving website change notifications straight away.
- See our [tutorials and how-to page for more inspiration](https://changedetection.io/tutorials) 
+

 ### Target specific parts of the webpage using the Visual Selector tool.

@@ -98,7 +97,7 @@ Please :star: star :star: this project and help it grow! https://github.com/dgtl
 With Docker composer, just clone this repository and..

 ```bash
-$ docker compose up -d
+$ docker-compose up -d
 ```

 Docker standalone
@@ -137,10 +136,10 @@ docker rm $(docker ps -a -f name=changedetection.io -q)
 docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
 ```

-### docker compose
+### docker-compose

 ```bash
-docker compose pull && docker compose up -d
+docker-compose pull && docker-compose up -d
 ```

 See the wiki for more information https://github.com/dgtlmoon/changedetection.io/wiki
@@ -0,0 +1,21 @@
+{
+  "name": "ChangeDetection.io",
+  "description": "The best and simplest self-hosted open source website change detection monitoring and notification service.",
+  "keywords": [
+    "changedetection",
+    "website monitoring"
+  ],
+  "repository": "https://github.com/dgtlmoon/changedetection.io",
+  "success_url": "/",
+  "scripts": {
+  },
+  "env": {
+  },
+  "formation": {
+    "web": {
+      "quantity": 1,
+      "size": "free"
+    }
+  },
+  "image": "heroku/python"
+}
@@ -1,6 +1,44 @@
 #!/usr/bin/python3

-# Only exists for direct CLI usage
+# Entry-point for running from the CLI when not installed via Pip, Pip will handle the console_scripts entry_points's from setup.py
+# It's recommended to use `pip3 install changedetection.io` and start with `changedetection.py` instead, it will be linkd to your global path.
+# or Docker.
+# Read more https://github.com/dgtlmoon/changedetection.io/wiki

-import changedetectionio
-changedetectionio.main()
+from changedetectionio import changedetection
+import multiprocessing
+import sys
+import os
+
+def sigchld_handler(_signo, _stack_frame):
+    import sys
+    print('Shutdown: Got SIGCHLD')
+    # https://stackoverflow.com/questions/40453496/python-multiprocessing-capturing-signals-to-restart-child-processes-or-shut-do
+    pid, status = os.waitpid(-1, os.WNOHANG | os.WUNTRACED | os.WCONTINUED)
+
+    print('Sub-process: pid %d status %d' % (pid, status))
+    if status != 0:
+        sys.exit(1)
+
+    raise SystemExit
+
+if __name__ == '__main__':
+
+    #signal.signal(signal.SIGCHLD, sigchld_handler)
+
+    # The only way I could find to get Flask to shutdown, is to wrap it and then rely on the subsystem issuing SIGTERM/SIGKILL
+    parse_process = multiprocessing.Process(target=changedetection.main)
+    parse_process.daemon = True
+    parse_process.start()
+    import time
+
+    try:
+        while True:
+            time.sleep(1)
+            if not parse_process.is_alive():
+                # Process died/crashed for some reason, exit with error set
+                sys.exit(1)
+
+    except KeyboardInterrupt:
+        #parse_process.terminate() not needed, because this process will issue it to the sub-process anyway
+        print ("Exited - CTRL+C")
@@ -280,14 +280,11 @@ class CreateWatch(Resource):
            if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()):
                continue

-            list[uuid] = {
-                'last_changed': watch.last_changed,
-                'last_checked': watch['last_checked'],
-                'last_error': watch['last_error'],
-                'title': watch['title'],
-                'url': watch['url'],
-                'viewed': watch.viewed
-            }
+            list[uuid] = {'url': watch['url'],
+                       'title': watch['title'],
+                       'last_checked': watch['last_checked'],
+                       'last_changed': watch.last_changed,
+                       'last_error': watch['last_error']}

        if request.args.get('recheck_all'):
            for uuid in self.datastore.data['watching'].keys():
@@ -296,61 +293,6 @@ class CreateWatch(Resource):

        return list, 200

-class Import(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-
-    @auth.check_token
-    def post(self):
-        """
-        @api {post} /api/v1/import Import a list of watched URLs
-        @apiDescription Accepts a line-feed separated list of URLs to import, additionally with ?tag_uuids=(tag  id), ?tag=(name), ?proxy={key}, ?dedupe=true (default true) one URL per line.
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/import --data-binary @list-of-sites.txt -H"x-api-key:8a111a21bc2f8f1dd9b9353bbd46049a"
-        @apiName Import
-        @apiGroup Watch
-        @apiSuccess (200) {List} OK List of watch UUIDs added
-        @apiSuccess (500) {String} ERR Some other error
-        """
-
-        extras = {}
-
-        if request.args.get('proxy'):
-            plist = self.datastore.proxy_list
-            if not request.args.get('proxy') in plist:
-                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
-            else:
-                extras['proxy'] = request.args.get('proxy')
-
-        dedupe = strtobool(request.args.get('dedupe', 'true'))
-
-        tags = request.args.get('tag')
-        tag_uuids = request.args.get('tag_uuids')
-
-        if tag_uuids:
-            tag_uuids = tag_uuids.split(',')
-
-        urls = request.get_data().decode('utf8').splitlines()
-        added = []
-        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
-        for url in urls:
-            url = url.strip()
-            if not len(url):
-                continue
-
-            # If hosts that only contain alphanumerics are allowed ("localhost" for example)
-            if not validators.url(url, simple_host=allow_simplehost):
-                return f"Invalid or unsupported URL - {url}", 400
-
-            if dedupe and self.datastore.url_exists(url):
-                continue
-
-            new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags, tag_uuids=tag_uuids)
-            added.append(new_uuid)
-
-        return added
-
 class SystemInfo(Resource):
    def __init__(self, **kwargs):
        # datastore is a black box dependency
@@ -25,9 +25,10 @@ from distutils.util import strtobool
 from flask import Blueprint, request, make_response
 import logging
 import os
+import re

 from changedetectionio.store import ChangeDetectionStore
-from changedetectionio.flask_app import login_optionally_required
+from changedetectionio import login_optionally_required

 browsersteps_sessions = {}
 io_interface_context = None
@@ -96,7 +97,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
            proxy=proxy)

        # For test
-        #browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time()))
+        #browsersteps_start_session['browserstepper'].action_goto_url(value="http://exbaseample.com?time="+str(time.time()))

        return browsersteps_start_session

@@ -151,7 +152,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
    @browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])
    def browsersteps_ui_update():
        import base64
-        import playwright._impl._errors
+        import playwright._impl._api_types
        global browsersteps_sessions
        from changedetectionio.blueprint.browser_steps import browser_steps

@@ -110,7 +110,7 @@ class steppable_browser_interface():
        self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))

    def action_click_element_if_exists(self, selector, value):
-        import playwright._impl._errors as _api_types
+        import playwright._impl._api_types as _api_types
        print("Clicking element if exists")
        if not len(selector.strip()):
            return
@@ -123,9 +123,6 @@ class steppable_browser_interface():
            return

    def action_click_x_y(self, selector, value):
-        if not re.match(r'^\s?\d+\s?,\s?\d+\s?$', value):
-            raise Exception("'Click X,Y' step should be in the format of '100 , 90'")
-
        x, y = value.strip().split(',')
        x = int(float(x.strip()))
        y = int(float(y.strip()))
@@ -41,7 +41,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
        now = time.time()
        try:
            update_handler = text_json_diff.perform_site_check(datastore=datastore, watch_uuid=uuid)
-            update_handler.call_browser()
+            update_handler.fetch_content()
        # title, size is len contents not len xfer
        except content_fetcher.Non200ErrorCodeReceived as e:
            if e.status_code == 404:
@@ -1,6 +1,6 @@
 from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
 from changedetectionio.store import ChangeDetectionStore
-from changedetectionio.flask_app import login_optionally_required
+from changedetectionio import login_optionally_required


 def construct_blueprint(datastore: ChangeDetectionStore):
@@ -3,7 +3,7 @@
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
 </script>

 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
@@ -0,0 +1,153 @@
+#!/usr/bin/python3
+
+# Launch as a eventlet.wsgi server instance.
+
+from distutils.util import strtobool
+from json.decoder import JSONDecodeError
+
+import eventlet
+import eventlet.wsgi
+import getopt
+import os
+import signal
+import socket
+import sys
+
+from . import store, changedetection_app, content_fetcher
+from . import __version__
+
+# Only global so we can access it in the signal handler
+app = None
+datastore = None
+
+def sigterm_handler(_signo, _stack_frame):
+    global app
+    global datastore
+#    app.config.exit.set()
+    print('Shutdown: Got SIGTERM, DB saved to disk')
+    datastore.sync_to_json()
+#    raise SystemExit
+
+def main():
+    global datastore
+    global app
+
+    datastore_path = None
+    do_cleanup = False
+    host = ''
+    ipv6_enabled = False
+    port = os.environ.get('PORT') or 5000
+    ssl_mode = False
+
+    # On Windows, create and use a default path.
+    if os.name == 'nt':
+        datastore_path = os.path.expandvars(r'%APPDATA%\changedetection.io')
+        os.makedirs(datastore_path, exist_ok=True)
+    else:
+        # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
+        datastore_path = os.path.join(os.getcwd(), "../datastore")
+
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:", "port")
+    except getopt.GetoptError:
+        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
+        sys.exit(2)
+
+    create_datastore_dir = False
+
+    for opt, arg in opts:
+        if opt == '-s':
+            ssl_mode = True
+
+        if opt == '-h':
+            host = arg
+
+        if opt == '-p':
+            port = int(arg)
+
+        if opt == '-d':
+            datastore_path = arg
+
+        if opt == '-6':
+            print ("Enabling IPv6 listen support")
+            ipv6_enabled = True
+
+        # Cleanup (remove text files that arent in the index)
+        if opt == '-c':
+            do_cleanup = True
+
+        # Create the datadir if it doesnt exist
+        if opt == '-C':
+            create_datastore_dir = True
+
+    # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
+    app_config = {'datastore_path': datastore_path}
+
+    if not os.path.isdir(app_config['datastore_path']):
+        if create_datastore_dir:
+            os.mkdir(app_config['datastore_path'])
+        else:
+            print(
+                "ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists or specify a directory with the -d option.\n"
+                "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
+            sys.exit(2)
+
+    try:
+        datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
+    except JSONDecodeError as e:
+        # Dont' start if the JSON DB looks corrupt
+        print ("ERROR: JSON DB or Proxy List JSON at '{}' appears to be corrupt, aborting".format(app_config['datastore_path']))
+        print(str(e))
+        return
+
+    app = changedetection_app(app_config, datastore)
+
+    signal.signal(signal.SIGTERM, sigterm_handler)
+
+    # Go into cleanup mode
+    if do_cleanup:
+        datastore.remove_unused_snapshots()
+
+    app.config['datastore_path'] = datastore_path
+
+
+    @app.context_processor
+    def inject_version():
+        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
+                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
+                    has_password=datastore.data['settings']['application']['password'] != False
+                    )
+
+    # Monitored websites will not receive a Referer header when a user clicks on an outgoing link.
+    # @Note: Incompatible with password login (and maybe other features) for now, submit a PR!
+    @app.after_request
+    def hide_referrer(response):
+        if strtobool(os.getenv("HIDE_REFERER", 'false')):
+            response.headers["Referrer-Policy"] = "no-referrer"
+
+        return response
+
+    # Proxy sub-directory support
+    # Set environment var USE_X_SETTINGS=1 on this script
+    # And then in your proxy_pass settings
+    #
+    #         proxy_set_header Host "localhost";
+    #         proxy_set_header X-Forwarded-Prefix /app;
+
+    if os.getenv('USE_X_SETTINGS'):
+        print ("USE_X_SETTINGS is ENABLED\n")
+        from werkzeug.middleware.proxy_fix import ProxyFix
+        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
+
+    s_type = socket.AF_INET6 if ipv6_enabled else socket.AF_INET
+
+    if ssl_mode:
+        # @todo finalise SSL config, but this should get you in the right direction if you need it.
+        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port), s_type),
+                                               certfile='cert.pem',
+                                               keyfile='privkey.pem',
+                                               server_side=True), app)
+
+    else:
+        eventlet.wsgi.server(eventlet.listen((host, int(port)), s_type), app)
+
@@ -43,11 +43,9 @@ class JSActionExceptions(Exception):
        return


-class BrowserStepsStepException(Exception):
-    def __init__(self, step_n, original_e):
+class BrowserStepsStepTimout(Exception):
+    def __init__(self, step_n):
        self.step_n = step_n
-        self.original_e = original_e
-        print(f"Browser Steps exception at step {self.step_n}", str(original_e))
        return


@@ -93,20 +91,19 @@ class ReplyWithContentButNoText(Exception):


 class Fetcher():
-    browser_connection_is_custom = None
-    browser_connection_url = None
    browser_steps = None
    browser_steps_screenshot_path = None
    content = None
    error = None
    fetcher_description = "No description"
+    browser_connection_url = None
    headers = {}
-    instock_data = None
-    instock_data_js = ""
    status_code = None
    webdriver_js_execute_code = None
    xpath_data = None
    xpath_element_js = ""
+    instock_data = None
+    instock_data_js = ""

    # Will be needed in the future by the VisualSelector, always get this where possible.
    screenshot = False
@@ -175,7 +172,7 @@ class Fetcher():

    def iterate_browser_steps(self):
        from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-        from playwright._impl._errors import TimeoutError, Error
+        from playwright._impl._api_types import TimeoutError
        from jinja2 import Environment
        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])

@@ -205,10 +202,10 @@ class Fetcher():
                                                      optional_value=optional_value)
                    self.screenshot_step(step_n)
                    self.save_step_html(step_n)
-
-                except (Error, TimeoutError) as e:
+                except TimeoutError as e:
+                    print(str(e))
                    # Stop processing here
-                    raise BrowserStepsStepException(step_n=step_n, original_e=e)
+                    raise BrowserStepsStepTimout(step_n=step_n)

    # It's always good to reset these
    def delete_browser_steps_screenshots(self):
@@ -255,19 +252,16 @@ class base_html_playwright(Fetcher):

    proxy = None

-    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+    def __init__(self, proxy_override=None, browser_connection_url=None):
        super().__init__()

        self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')

-        if custom_browser_connection_url:
-            self.browser_connection_is_custom = True
-            self.browser_connection_url = custom_browser_connection_url
-        else:
-            # Fallback to fetching from system
-            # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+        if not browser_connection_url:
            self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
-
+        else:
+            self.browser_connection_url = browser_connection_url

        # If any proxy settings are enabled, then we should setup the proxy object
        proxy_args = {}
@@ -427,10 +421,8 @@ class base_html_playwright(Fetcher):
            current_include_filters=None,
            is_binary=False):

-
        # For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!)
-        # browser_connection_is_custom doesnt work with puppeteer style fetch (use playwright native too in this case)
-        if not self.browser_connection_is_custom and not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
+        if not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
            if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')):
                # Temporary backup solution until we rewrite the playwright code
                return self.run_fetch_browserless_puppeteer(
@@ -444,7 +436,7 @@ class base_html_playwright(Fetcher):
                    is_binary)

        from playwright.sync_api import sync_playwright
-        import playwright._impl._errors
+        import playwright._impl._api_types

        self.delete_browser_steps_screenshots()
        response = None
@@ -497,7 +489,7 @@ class base_html_playwright(Fetcher):
            try:
                if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
                    browsersteps_interface.action_execute_js(value=self.webdriver_js_execute_code, selector=None)
-            except playwright._impl._errors.TimeoutError as e:
+            except playwright._impl._api_types.TimeoutError as e:
                context.close()
                browser.close()
                # This can be ok, we will try to grab what we could retrieve
@@ -577,16 +569,15 @@ class base_html_webdriver(Fetcher):
                                        'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
    proxy = None

-    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+    def __init__(self, proxy_override=None, browser_connection_url=None):
        super().__init__()
        from selenium.webdriver.common.proxy import Proxy as SeleniumProxy

        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-        if not custom_browser_connection_url:
+        if not browser_connection_url:
            self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
        else:
-            self.browser_connection_is_custom = True
-            self.browser_connection_url = custom_browser_connection_url
+            self.browser_connection_url = browser_connection_url

        # If any proxy settings are enabled, then we should setup the proxy object
        proxy_args = {}
@@ -683,7 +674,7 @@ class base_html_webdriver(Fetcher):
 class html_requests(Fetcher):
    fetcher_description = "Basic fast Plaintext/HTTP Client"

-    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+    def __init__(self, proxy_override=None, browser_connection_url=None):
        super().__init__()
        self.proxy_override = proxy_override
        # browser_connection_url is none because its always 'launched locally'
@@ -38,7 +38,6 @@ base_config = {
    'track_ldjson_price_data': None,
    'headers': {},  # Extra headers to send
    'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
-    'in_stock' : None,
    'in_stock_only' : True, # Only trigger change on going to instock from out-of-stock
    'include_filters': [],
    'last_checked': 0,
@@ -114,8 +113,7 @@ class model(dict):

    @property
    def viewed(self):
-        # Don't return viewed when last_viewed is 0 and newest_key is 0
-        if int(self['last_viewed']) and int(self['last_viewed']) >= int(self.newest_history_key) :
+        if int(self['last_viewed']) >= int(self.newest_history_key) :
            return True

        return False
@@ -46,9 +46,6 @@ from apprise.decorators import notify
@notify(on="puts")
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
    import requests
-    from apprise.utils import parse_url as apprise_parse_url
-    from apprise.URLBase import URLBase
-
    url = kwargs['meta'].get('url')

    if url.startswith('post'):
@@ -71,45 +68,16 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
    url = url.replace('delete://', 'http://')
    url = url.replace('deletes://', 'https://')

-    headers = {}
-    params = {}
-    auth = None
-
-    # Convert /foobar?+some-header=hello to proper header dictionary
-    results = apprise_parse_url(url)
-    if results:
-        # Add our headers that the user can potentially over-ride if they wish
-        # to to our returned result set and tidy entries by unquoting them
-        headers = {URLBase.unquote(x): URLBase.unquote(y)
-                   for x, y in results['qsd+'].items()}
-
-        # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-        # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
-        # but here we are making straight requests, so we need todo convert this against apprise's logic
-        for k, v in results['qsd'].items():
-            if not k.strip('+-') in results['qsd+'].keys():
-                params[URLBase.unquote(k)] = URLBase.unquote(v)
-
-        # Determine Authentication
-        auth = ''
-        if results.get('user') and results.get('password'):
-            auth = (URLBase.unquote(results.get('user')), URLBase.unquote(results.get('user')))
-        elif results.get('user'):
-            auth = (URLBase.unquote(results.get('user')))
-
    # Try to auto-guess if it's JSON
+    headers = {}
    try:
        json.loads(body)
-        headers['Content-Type'] = 'application/json; charset=utf-8'
+        headers = {'Content-Type': 'application/json; charset=utf-8'}
    except ValueError as e:
        pass

-    r(results.get('url'),
-      auth=auth,
-      data=body,
-      headers=headers,
-      params=params
-      )
+
+    r(url, headers=headers, data=body)


 def process_notification(n_object, datastore):
@@ -119,8 +87,8 @@ def process_notification(n_object, datastore):

    # Get the notification body from datastore
    jinja2_env = Environment(loader=BaseLoader)
-    n_body = jinja2_env.from_string(n_object.get('notification_body', '')).render(**notification_parameters)
-    n_title = jinja2_env.from_string(n_object.get('notification_title', '')).render(**notification_parameters)
+    n_body = jinja2_env.from_string(n_object.get('notification_body', default_notification_body)).render(**notification_parameters)
+    n_title = jinja2_env.from_string(n_object.get('notification_title', default_notification_title)).render(**notification_parameters)
    n_format = valid_notification_formats.get(
        n_object.get('notification_format', default_notification_format),
        valid_notification_formats[default_notification_format],
@@ -187,8 +155,8 @@ def process_notification(n_object, datastore):
                    if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
                        prefix = '?' if not '?' in url else '&'
                        # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
-                        n_format = n_format.lower()
-                        url = f"{url}{prefix}format={n_format}"
+                        n_format = n_format.tolower()
+                        url = "{}{}format={}".format(url, prefix, n_format)
                    # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only

                apobj.add(url)
@@ -221,14 +189,13 @@ def process_notification(n_object, datastore):


 # Notification title + body content parameters get created here.
-# ( Where we prepare the tokens in the notification to be replaced with actual values )
 def create_notification_parameters(n_object, datastore):
    from copy import deepcopy

    # in the case we send a test notification from the main settings, there is no UUID.
    uuid = n_object['uuid'] if 'uuid' in n_object else ''

-    if uuid:
+    if uuid != '':
        watch_title = datastore.data['watching'][uuid].get('title', '')
        tag_list = []
        tags = datastore.get_all_tags_for_watch(uuid)
@@ -256,7 +223,7 @@ def create_notification_parameters(n_object, datastore):
    tokens.update(
        {
            'base_url': base_url,
-            'current_snapshot': n_object.get('current_snapshot', ''),
+            'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '',
            'diff': n_object.get('diff', ''),  # Null default in the case we use a test
            'diff_added': n_object.get('diff_added', ''),  # Null default in the case we use a test
            'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test
@@ -4,10 +4,8 @@ import hashlib
 import re
 from changedetectionio import content_fetcher
 from copy import deepcopy
-from distutils.util import strtobool
-
-class difference_detection_processor():

+class difference_detection_processor_interface():
    browser_steps = None
    datastore = None
    fetcher = None
@@ -15,57 +13,36 @@ class difference_detection_processor():
    watch = None
    xpath_data = None

-    def __init__(self, *args, datastore, watch_uuid, **kwargs):
-        super().__init__(*args, **kwargs)
+
+    @abstractmethod
+    def run_changedetection(self, uuid, skip_when_checksum_same=True):
+        update_obj = {'last_notification_error': False, 'last_error': False}
+        some_data = 'xxxxx'
+        update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()
+        changed_detected = False
+        return changed_detected, update_obj, ''.encode('utf-8')
+
+
+class text_content_difference_detection_processor(difference_detection_processor_interface):
+
+    def __init__(self, *args, datastore, watch_uuid, prefer_fetch_backend, **kwargs):
        self.datastore = datastore
        self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
+        self.prefer_fetch_backend = prefer_fetch_backend
+        super().__init__(*args, **kwargs)

-    def call_browser(self):
-
-        # Protect against file:// access
-        if re.search(r'^file://', self.watch.get('url', '').strip(), re.IGNORECASE):
-            if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
-                raise Exception(
-                    "file:// type access is denied for security reasons."
-                )
-
-        url = self.watch.link
-
-        # Requests, playwright, other browser via wss:// etc, fetch_extra_something
-        prefer_fetch_backend = self.watch.get('fetch_backend', 'system')
-
-        # Proxy ID "key"
-        preferred_proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=self.watch.get('uuid'))
-
-        # Pluggable content self.fetcher
-        if not prefer_fetch_backend or prefer_fetch_backend == 'system':
-            prefer_fetch_backend = self.datastore.data['settings']['application'].get('fetch_backend')
-
-        # In the case that the preferred fetcher was a browser config with custom connection URL..
-        # @todo - on save watch, if its extra_browser_ then it should be obvious it will use playwright (like if its requests now..)
-        custom_browser_connection_url = None
-        if prefer_fetch_backend.startswith('extra_browser_'):
-            (t, key) = prefer_fetch_backend.split('extra_browser_')
-            connection = list(
-                filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', [])))
-            if connection:
-                prefer_fetch_backend = 'base_html_playwright'
-                custom_browser_connection_url = connection[0].get('browser_connection_url')
-
-        # PDF should be html_requests because playwright will serve it up (so far) in a embedded page
-        # @todo https://github.com/dgtlmoon/changedetection.io/issues/2019
-        # @todo needs test to or a fix
-        if self.watch.is_pdf:
-           prefer_fetch_backend = "html_requests"
-
+        ########################################
+        # Attach the correct fetcher and proxy #
+        ########################################
        # Grab the right kind of 'fetcher', (playwright, requests, etc)
-        if hasattr(content_fetcher, prefer_fetch_backend):
-            fetcher_obj = getattr(content_fetcher, prefer_fetch_backend)
+        if hasattr(content_fetcher, self.prefer_fetch_backend):
+            fetcher_obj = getattr(content_fetcher, self.prefer_fetch_backend)
        else:
            # If the klass doesnt exist, just use a default
            fetcher_obj = getattr(content_fetcher, "html_requests")

-
+        # Proxy ID "key"
+        preferred_proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=self.watch.get('uuid'))
        proxy_url = None
        if preferred_proxy_id:
            proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')
@@ -74,9 +51,23 @@ class difference_detection_processor():
        # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
        # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
        self.fetcher = fetcher_obj(proxy_override=proxy_url,
-                                   custom_browser_connection_url=custom_browser_connection_url
+                                   browser_connection_url=None # Default, let each fetcher work it out
                                   )

+    def fetch_content(self):
+
+        url = self.watch.link
+
+        # In the case that the preferred fetcher was a browser config with custom connection URL..
+        # @todo - on save watch, if its extra_browser_ then it should be obvious it will use playwright (like if its requests now..)
+        if self.prefer_fetch_backend.startswith('extra_browser_'):
+            (t, key) = self.prefer_fetch_backend.split('extra_browser_')
+            connection = list(
+                filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', [])))
+            if connection:
+                prefer_fetch_backend = 'base_html_playwright'
+                browser_connection_url = connection[0].get('browser_connection_url')
+
        if self.watch.has_browser_steps:
            self.fetcher.browser_steps = self.watch.get('browser_steps', [])
            self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid'))
@@ -120,14 +111,6 @@ class difference_detection_processor():

        # After init, call run_changedetection() which will do the actual change-detection

-    @abstractmethod
-    def run_changedetection(self, uuid, skip_when_checksum_same=True):
-        update_obj = {'last_notification_error': False, 'last_error': False}
-        some_data = 'xxxxx'
-        update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()
-        changed_detected = False
-        return changed_detected, update_obj, ''.encode('utf-8')
-

 def available_processors():
    from . import restock_diff, text_json_diff
@@ -1,8 +1,9 @@

 import hashlib
 import urllib3
-from . import difference_detection_processor
+#from . import browser_content_difference_detection_processor
 from copy import deepcopy
+from . import text_content_difference_detection_processor

 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

@@ -15,7 +16,7 @@ class UnableToExtractRestockData(Exception):
        self.status_code = status_code
        return

-class perform_site_check(difference_detection_processor):
+class perform_site_check(text_content_difference_detection_processor):
    screenshot = None
    xpath_data = None

@@ -10,8 +10,8 @@ import urllib3
 from changedetectionio import content_fetcher, html_tools
 from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
 from copy import deepcopy
-from . import difference_detection_processor
 from ..html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text
+from . import text_content_difference_detection_processor

 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

@@ -31,7 +31,7 @@ class PDFToHTMLToolNotFound(ValueError):

 # Some common stuff here that can be moved to a base class
 # (set_proxy_from_list)
-class perform_site_check(difference_detection_processor):
+class perform_site_check(text_content_difference_detection_processor):

    def run_changedetection(self, uuid, skip_when_checksum_same=True):
        changed_detected = False
@@ -1,10 +1,8 @@
 function isItemInStock() {
  // @todo Pass these in so the same list can be used in non-JS fetchers
  const outOfStockTexts = [
-    ' أخبرني عندما يتوفر',
    '0 in stock',
    'agotado',
-    'article épuisé',
    'artikel zurzeit vergriffen',
    'as soon as stock is available',
    'ausverkauft', // sold out
@@ -21,14 +19,11 @@ function isItemInStock() {
    'en rupture de stock',
    'ist derzeit nicht auf lager',
    'item is no longer available',
-    'let me know when it\'s available',
    'message if back in stock',
    'nachricht bei',
    'nicht auf lager',
    'nicht lieferbar',
    'nicht zur verfügung',
-    'niet beschikbaar',
-    'niet leverbaar',
    'no disponible temporalmente',
    'no longer in stock',
    'no tickets available',
@@ -41,17 +36,13 @@ function isItemInStock() {
    'out-of-stock',
    'produkt niedostępny',
    'sold out',
-    'sold-out',
    'temporarily out of stock',
    'temporarily unavailable',
    'tickets unavailable',
-    'tijdelijk uitverkocht',
    'unavailable tickets',
    'we do not currently have an estimate of when this product will be back in stock.',
    'zur zeit nicht an lager',
-    '品切れ',
    '已售完',
-    '품절'
  ];


@@ -114,4 +105,4 @@ function isItemInStock() {
 }

 // returns the element text that makes it think it's out of stock
-return isItemInStock();
+return isItemInStock();
@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg width="15" height="16.363636" viewBox="0 0 15 16.363636" xmlns="http://www.w3.org/2000/svg" >
+<svg width="15" height="16.363636" viewBox="0 0 15 16.363636" xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg">
  <path d="m 14.318182,11.762045 v 1.1925 H 5.4102273 L 11.849318,7.1140909 C 12.234545,9.1561364 12.54,11.181818 14.318182,11.762045 Z m -6.7984093,4.601591 c 1.0759091,0 2.0256823,-0.955909 2.0256823,-2.045454 H 5.4545455 c 0,1.089545 0.9879545,2.045454 2.0652272,2.045454 z M 15,2.8622727 0.9177273,15.636136 0,14.627045 l 1.8443182,-1.6725 h -1.1625 v -1.1925 C 4.0070455,10.677273 2.1784091,4.5388636 5.3611364,2.6897727 5.8009091,2.4347727 6.0709091,1.9609091 6.0702273,1.4488636 v -0.00205 C 6.0702273,0.64772727 6.7104545,0 7.5,0 8.2895455,0 8.9297727,0.64772727 8.9297727,1.4468182 v 0.00205 C 8.9290909,1.9602319 9.199773,2.4354591 9.638864,2.6897773 10.364318,3.111141 10.827273,3.7568228 11.1525,4.5129591 L 14.085682,1.8531818 Z M 6.8181818,1.3636364 C 6.8181818,1.74 7.1236364,2.0454545 7.5,2.0454545 7.8763636,2.0454545 8.1818182,1.74 8.1818182,1.3636364 8.1818182,0.98795455 7.8763636,0.68181818 7.5,0.68181818 c -0.3763636,0 -0.6818182,0.30613637 -0.6818182,0.68181822 z" id="path2" style="fill:#f8321b;stroke-width:0.681818;fill-opacity:1"/>
 </svg>
@@ -10,7 +10,7 @@
   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
   xmlns="http://www.w3.org/2000/svg"
-   >
+   xmlns:svg="http://www.w3.org/2000/svg">
  <defs
     id="defs16" />
  <sodipodi:namedview
@@ -12,7 +12,7 @@
   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
   xmlns="http://www.w3.org/2000/svg"
-   ><defs
+   xmlns:svg="http://www.w3.org/2000/svg"><defs
     id="defs11" /><sodipodi:namedview
     id="namedview9"
     pagecolor="#ffffff"
@@ -10,7 +10,7 @@
   viewBox="0 0 7.1975545 4.7993639"
   xml:space="preserve"
   xmlns="http://www.w3.org/2000/svg"
-   ><defs
+   xmlns:svg="http://www.w3.org/2000/svg"><defs
   id="defs19" />
 <g
   id="g14"
@@ -9,7 +9,7 @@
   id="svg5"
   xmlns:xlink="http://www.w3.org/1999/xlink"
   xmlns="http://www.w3.org/2000/svg"
-   >
+   xmlns:svg="http://www.w3.org/2000/svg">
  <defs
     id="defs2" />
  <g
@@ -10,7 +10,7 @@
   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
   xmlns="http://www.w3.org/2000/svg"
-   >
+   xmlns:svg="http://www.w3.org/2000/svg">
  <defs
     id="defs12" />
  <sodipodi:namedview
@@ -3,6 +3,7 @@
   xmlns:dc="http://purl.org/dc/elements/1.1/"
   xmlns:cc="http://creativecommons.org/ns#"
   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
   xmlns="http://www.w3.org/2000/svg"
   version="1.1"
   id="Capa_1"
@@ -13,6 +13,7 @@
   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg"
   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
   xmlns:cc="http://creativecommons.org/ns#"
   xmlns:dc="http://purl.org/dc/elements/1.1/"><sodipodi:namedview
@@ -6,7 +6,7 @@
   version="1.1"
   id="svg6"
   xmlns="http://www.w3.org/2000/svg"
-   >
+   xmlns:svg="http://www.w3.org/2000/svg">
  <defs
     id="defs10" />
  <path
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg width="18" height="19.92" viewBox="0 0 18 19.92" xmlns="http://www.w3.org/2000/svg" >
+<svg width="18" height="19.92" viewBox="0 0 18 19.92" xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg">
  <path d="M -3,-2 H 21 V 22 H -3 Z" fill="none" id="path2"/>
  <path d="m 15,14.08 c -0.76,0 -1.44,0.3 -1.96,0.77 L 5.91,10.7 C 5.96,10.47 6,10.24 6,10 6,9.76 5.96,9.53 5.91,9.3 L 12.96,5.19 C 13.5,5.69 14.21,6 15,6 16.66,6 18,4.66 18,3 18,1.34 16.66,0 15,0 c -1.66,0 -3,1.34 -3,3 0,0.24 0.04,0.47 0.09,0.7 L 5.04,7.81 C 4.5,7.31 3.79,7 3,7 1.34,7 0,8.34 0,10 c 0,1.66 1.34,3 3,3 0.79,0 1.5,-0.31 2.04,-0.81 l 7.12,4.16 c -0.05,0.21 -0.08,0.43 -0.08,0.65 0,1.61 1.31,2.92 2.92,2.92 1.61,0 2.92,-1.31 2.92,-2.92 0,-1.61 -1.31,-2.92 -2.92,-2.92 z" id="path4" style="fill:#0078e7;fill-opacity:1"/>
 </svg>
@@ -1,4 +1,19 @@
 $(document).ready(function () {
+    function toggle() {
+        if ($('input[name="application-fetch_backend"]:checked').val() != 'html_requests') {
+            $('#requests-override-options').hide();
+            $('#webdriver-override-options').show();
+        } else {
+            $('#requests-override-options').show();
+            $('#webdriver-override-options').hide();
+        }
+    }
+
+    $('input[name="application-fetch_backend"]').click(function (e) {
+        toggle();
+    });
+    toggle();
+
    $("#api-key").hover(
        function () {
            $("#api-key-copy").html('copy').fadeIn();
@@ -24,17 +24,14 @@ $(document).ready(function() {
    })

    data = {
-      notification_body: $('#notification_body').val(),
-      notification_format: $('#notification_format').val(),
-      notification_title: $('#notification_title').val(),
-      notification_urls: $('.notification-urls').val(),
-      window_url: window.location.href,
+        window_url : window.location.href,
+        notification_urls : $('.notification-urls').val(),
    }
-
-
-    if (!data['notification_urls'].length) {
-      alert("Notification URL list is empty, cannot send test.")
-      return;
+    for (key in data) {
+      if (!data[key].length) {
+        alert(key+" is empty, cannot send test.")
+        return;
+      }
    }

    $.ajax({
@@ -1,29 +0,0 @@
-$(document).ready(function () {
-
-    // Lazy Hide/Show elements mechanism
-    $('[data-visible-for]').hide();
-    function show_related_elem(e) {
-        var n = $(e).attr('name') + "=" + $(e).val();
-        if (n === 'fetch_backend=system') {
-            n = "fetch_backend=" + default_system_fetch_backend;
-        }
-        $(`[data-visible-for~="${n}"]`).show();
-    }
-    $(':radio').on('keyup keypress blur change click', function (e) {
-        $(`[data-visible-for]`).hide();
-        $('.advanced-options').hide();
-        show_related_elem(this);
-    });
-
-    $(':radio:checked').each(function (e) {
-       show_related_elem(this);
-    })
-
-
-    // Show advanced
-    $('.show-advanced').click(function (e) {
-        $(this).closest('.tab-pane-inner').find('.advanced-options').each(function (e) {
-            $(this).toggle();
-        })
-    });
-});
@@ -1,4 +1,40 @@
 $(document).ready(function () {
+    function toggle() {
+        if ($('input[name="fetch_backend"]:checked').val() == 'html_webdriver') {
+            if (playwright_enabled) {
+                // playwright supports headers, so hide everything else
+                // See #664
+                $('#requests-override-options #request-method').hide();
+                $('#requests-override-options #request-body').hide();
+
+                // @todo connect this one up
+                $('#ignore-status-codes-option').hide();
+            } else {
+                // selenium/webdriver doesnt support anything afaik, hide it all
+                $('#requests-override-options').hide();
+            }
+
+            $('#webdriver-override-options').show();
+
+        } else if ($('input[name="fetch_backend"]:checked').val() == 'system') {
+            $('#requests-override-options #request-method').hide();
+            $('#requests-override-options #request-body').hide();
+            $('#ignore-status-codes-option').hide();
+            $('#requests-override-options').hide();
+            $('#webdriver-override-options').hide();
+        } else {
+
+            $('#requests-override-options').show();
+            $('#requests-override-options *:hidden').show();
+            $('#webdriver-override-options').hide();
+        }
+    }
+
+    $('input[name="fetch_backend"]').click(function (e) {
+        toggle();
+    });
+    toggle();
+
    $('#notification-setting-reset-to-default').click(function (e) {
        $('#notification_title').val('');
        $('#notification_body').val('');
@@ -402,24 +402,8 @@ label {
  }

  #watch-add-wrapper-zone {
-
-    @media only screen and (min-width: 760px) {
-      display: flex;
-      gap: 0.3rem;
-      flex-direction: row;
-    }
-    /* URL field grows always, other stay static in width */
-    > span {
-      flex-grow: 0;
-
-      input {
-        width: 100%;
-        padding-right: 1em;
-      }
-
-      &:first-child {
-        flex-grow: 1;
-      }
+    >div {
+      display: inline-block;
    }

    @media only screen and (max-width: 760px) {
@@ -960,8 +944,10 @@ ul {

@import "parts/_visualselector";

-#webdriver_delay {
+#webdriver-override-options {
+  input[type="number"] {
    width: 5em;
+  }
 }

 #api-key {
@@ -683,23 +683,11 @@ label:hover {
  #new-watch-form legend {
    color: var(--color-text-legend);
    font-weight: bold; }
-  #new-watch-form #watch-add-wrapper-zone {
-    /* URL field grows always, other stay static in width */ }
-    @media only screen and (min-width: 760px) {
-      #new-watch-form #watch-add-wrapper-zone {
-        display: flex;
-        gap: 0.3rem;
-        flex-direction: row; } }
-    #new-watch-form #watch-add-wrapper-zone > span {
-      flex-grow: 0; }
-      #new-watch-form #watch-add-wrapper-zone > span input {
-        width: 100%;
-        padding-right: 1em; }
-      #new-watch-form #watch-add-wrapper-zone > span:first-child {
-        flex-grow: 1; }
-    @media only screen and (max-width: 760px) {
-      #new-watch-form #watch-add-wrapper-zone #url {
-        width: 100%; } }
+  #new-watch-form #watch-add-wrapper-zone > div {
+    display: inline-block; }
+  @media only screen and (max-width: 760px) {
+    #new-watch-form #watch-add-wrapper-zone #url {
+      width: 100%; } }

 #diff-col {
  padding-left: 40px; }
@@ -1077,7 +1065,7 @@ ul {
 #selector-current-xpath {
  font-size: 80%; }

-#webdriver_delay {
+#webdriver-override-options input[type="number"] {
  width: 5em; }

 #api-key:hover {
@@ -234,7 +234,7 @@ class ChangeDetectionStore:

        # Probably their should be dict...
        for watch in self.data['watching'].values():
-            if watch['url'].lower() == url.lower():
+            if watch['url'] == url:
                return True

        return False
@@ -248,7 +248,6 @@ class ChangeDetectionStore:
                'check_count': 0,
                'fetch_time' : 0.0,
                'has_ldjson_price_data': None,
-                'in_stock': None,
                'last_checked': 0,
                'last_error': False,
                'last_notification_error': False,
@@ -334,8 +333,7 @@ class ChangeDetectionStore:

        # Or if UUIDs given directly
        if tag_uuids:
-            for t in tag_uuids:
-                apply_extras['tags'] = list(set(apply_extras['tags'] + [t.strip()]))
+            apply_extras['tags'] = list(set(apply_extras['tags'] + tag_uuids))

        # Make any uuids unique
        if apply_extras.get('tags'):
@@ -13,10 +13,10 @@
                            <div class="pure-form-message-inline">
                              <ul>
                                <li>Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.</li>
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) </code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
-                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>) <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes#postposts">more help here</a></li>
+                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
                                  <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                              </ul>
                            </div>
@@ -39,24 +39,6 @@
 {% endmacro %}


-{% macro render_nolabel_field(field) %}
-    <span>
-    {{ field(**kwargs)|safe }}
-        {% if field.errors %}
-            <span class="error">
-      {% if field.errors %}
-          <ul class=errors>
-        {% for error in field.errors %}
-            <li>{{ error }}</li>
-        {% endfor %}
-        </ul>
-      {% endif %}
-      </span>
-        {% endif %}
-    </span>
-{% endmacro %}
-
-
 {% macro render_button(field) %}
  {{ field(**kwargs)|safe }}
 {% endmacro %}
@@ -116,7 +116,7 @@
                   viewBox="0 0 16.9 16.1"
                   id="svg-heart"
                   xmlns="http://www.w3.org/2000/svg"
-                   >
+                   xmlns:svg="http://www.w3.org/2000/svg">
                  <path id="heartpath" d="M 5.338316,0.50302766 C 0.71136983,0.50647126 -3.9576371,7.2707777 8.5004254,15.503028 23.833425,5.3700277 13.220206,-2.5384409 8.6762066,1.6475589 c -0.060791,0.054322 -0.11943,0.1110064 -0.1757812,0.1699219 -0.057,-0.059 -0.1157813,-0.116875 -0.1757812,-0.171875 C 7.4724566,0.86129334 6.4060729,0.50223298 5.338316,0.50302766 Z"
                     style="fill:var(--color-background);fill-opacity:1;stroke:#ff0000;stroke-opacity:1" />
                </svg>
@@ -170,6 +170,7 @@
                    And tell your friends and colleagues :)
                </li>
                </ul>
+                </p>
                <p>
                    The more popular changedetection.io is, the more time we can dedicate to adding amazing features!
                </p>
@@ -3,7 +3,6 @@
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
-<script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script>
    const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
    const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
@@ -14,13 +13,13 @@
 {% if emailprefix %}
    const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
 {% endif %}
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
    const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
    const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
    const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
    const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
-    const default_system_fetch_backend="{{ settings_application['fetch_backend'] }}";
+
 </script>

 <script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
@@ -125,9 +124,10 @@
                        </span>
                    </div>
                {% endif %}
-
-                <!-- webdriver always -->
-                <fieldset data-visible-for="fetch_backend=html_webdriver"  style="display: none;">
+                    <div  class="pure-control-group inline-radio">
+                        {{ render_checkbox_field(form.ignore_status_codes) }}
+                    </div>
+                <fieldset id="webdriver-override-options">
                    <div class="pure-control-group">
                        {{ render_field(form.webdriver_delay) }}
                        <div class="pure-form-message-inline">
@@ -140,40 +140,23 @@
                        </div>
                    </div>
                    <div class="pure-control-group">
-                        <a class="pure-button button-secondary button-xsmall show-advanced">Show advanced options</a>
-                    </div>
-                    <div class="advanced-options"  style="display: none;">
                        {{ render_field(form.webdriver_js_execute_code) }}
                        <div class="pure-form-message-inline">
-                            Run this code before performing change detection, handy for filling in fields and other
-                            actions <a
-                                href="https://github.com/dgtlmoon/changedetection.io/wiki/Run-JavaScript-before-change-detection">More
-                            help and examples here</a>
+                            Run this code before performing change detection, handy for filling in fields and other actions <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Run-JavaScript-before-change-detection">More help and examples here</a>
                        </div>
                    </div>
                </fieldset>
-                <!-- html requests always -->
-                <fieldset data-visible-for="fetch_backend=html_requests">
-                    <div class="pure-control-group">
-                        <a class="pure-button button-secondary button-xsmall show-advanced">Show advanced options</a>
-                    </div>
-                    <div class="advanced-options"  style="display: none;">
-                        <div class="pure-control-group" id="request-method">
-                            {{ render_field(form.method) }}
-                        </div>
-                        <div id="request-body">
-                                            {{ render_field(form.body, rows=5, placeholder="Example
-{
-   \"name\":\"John\",
-   \"age\":30,
-   \"car\":null
-}") }}
+                <fieldset class="pure-group" id="requests-override-options">
+                    {% if not playwright_enabled %}
+                        <div class="pure-form-message-inline">
+                            <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
                        </div>
+                    {% endif %}
+                    <div class="pure-control-group" id="request-method">
+                        {{ render_field(form.method) }}
                    </div>
-                </fieldset>
-            <!-- hmm -->
-                <div class="pure-control-group advanced-options"  style="display: none;">
-                    {{ render_field(form.headers, rows=5, placeholder="Example
+                    <div class="pure-control-group" id="request-headers">
+{{ render_field(form.headers, rows=5, placeholder="Example
 Cookie: foobar
 User-Agent: wonderbra 1.0") }}

@@ -186,12 +169,17 @@ User-Agent: wonderbra 1.0") }}
                            <br>
                            (Not supported by Selenium browser)
                        </div>
+
                    </div>
-            <fieldset data-visible-for="fetch_backend=html_requests fetch_backend=html_webdriver" >
-                    <div class="pure-control-group inline-radio advanced-options"  style="display: none;">
-                    {{ render_checkbox_field(form.ignore_status_codes) }}
+                    <div class="pure-control-group" id="request-body">
+                                        {{ render_field(form.body, rows=5, placeholder="Example
+{
+   \"name\":\"John\",
+   \"age\":30,
+   \"car\":null
+}") }}
                    </div>
-            </fieldset>
+                </fieldset>
            </div>
            {% if playwright_enabled %}
            <div class="tab-pane-inner" id="browser-steps">
@@ -4,14 +4,14 @@
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
 {% if emailprefix %}
    const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
 {% endif %}
 </script>
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
-<script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
+
 <script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
 <div class="edit-form">
    <div class="tabs collapsable">
@@ -111,7 +111,7 @@
                    <br>
                    Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
                </div>
-                <fieldset class="pure-group" id="webdriver-override-options" data-visible-for="application-fetch_backend=html_webdriver">
+                <fieldset class="pure-group" id="webdriver-override-options">
                    <div class="pure-form-message-inline">
                        <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
                        <br>
@@ -236,11 +236,8 @@ nav
                <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
                </div>
                <div class="pure-control-group" id="extra-browsers-setting">
-                    <p>
-                    <span class="pure-form-message-inline"><i>Extra Browsers</i> can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.</span><br>
-                    <span class="pure-form-message-inline">Simply paste the connection address into the box, <a href="https://changedetection.io/tutorial/using-bright-datas-scraping-browser-pass-captchas-and-other-protection-when-monitoring">More instructions and examples here</a> </span>
-                    </p>
-                    {{ render_field(form.requests.form.extra_browsers) }}
+                    <span class="pure-form-message-inline"><i>Extra Browsers</i> allow changedetection.io to communicate with a different web-browser.</span><br>
+                  {{ render_field(form.requests.form.extra_browsers) }}
                </div>
            </div>
            <div id="actions">
@@ -1 +1 @@
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>
+<?xml version="1.0" encoding="utf-8"?><svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>
@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.jinja' import render_simple_field, render_field, render_nolabel_field %}
+{% from '_helpers.jinja' import render_simple_field, render_field %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>

@@ -11,14 +11,17 @@
        <fieldset>
            <legend>Add a new change detection watch</legend>
            <div id="watch-add-wrapper-zone">
-
-                    {{ render_nolabel_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_nolabel_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
-                    {{ render_nolabel_field(form.watch_submit_button, title="Watch this URL!" ) }}
-                    {{ render_nolabel_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
+                <div>
+                    {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
+                    {{ render_simple_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
+                </div>
+                <div>
+                    {{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
+                    {{ render_simple_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
+                </div>
            </div>
            <div id="quick-watch-processor-type">
-                {{ render_simple_field(form.processor) }}
+                {{ render_simple_field(form.processor, title="Edit first then Watch") }}
            </div>

        </fieldset>
@@ -141,7 +144,7 @@
                    {% if watch['processor'] == 'restock_diff'  %}
                    <span class="restock-label {{'in-stock' if watch['in_stock'] else 'not-in-stock' }}" title="detecting restock conditions">
                        <!-- maybe some object watch['processor'][restock_diff] or.. -->
-                        {% if watch['last_checked'] and watch['in_stock'] != None %}
+                        {% if watch['last_checked'] %}
                            {% if watch['in_stock'] %} In stock {% else %} Not in stock {% endif %}
                        {% else %}
                            Not yet checked
@@ -13,17 +13,22 @@ global app


 def cleanup(datastore_path):
-    import glob
    # Unlink test output files
-
-    for g in ["*.txt", "*.json", "*.pdf"]:
-        files = glob.glob(os.path.join(datastore_path, g))
-        for f in files:
-            if 'proxies.json' in f:
-                # Usually mounted by docker container during test time
-                continue
-            if os.path.isfile(f):
-                os.unlink(f)
+    files = [
+        'count.txt',
+        'endpoint-content.txt'
+        'headers.txt',
+        'headers-testtag.txt',
+        'notification.txt',
+        'secret.txt',
+        'url-watches.json',
+        'output.txt',
+    ]
+    for file in files:
+        try:
+            os.unlink("{}/{}".format(datastore_path, file))
+        except FileNotFoundError:
+            pass

@pytest.fixture(scope='session')
 def app(request):
@@ -96,9 +96,7 @@ def test_api_simple(client, live_server):
    )
    assert watch_uuid in res.json.keys()
    before_recheck_info = res.json[watch_uuid]
-
    assert before_recheck_info['last_checked'] != 0
-
    #705 `last_changed` should be zero on the first check
    assert before_recheck_info['last_changed'] == 0
    assert before_recheck_info['title'] == 'My test URL'
@@ -159,18 +157,6 @@ def test_api_simple(client, live_server):
    # @todo how to handle None/default global values?
    assert watch['history_n'] == 2, "Found replacement history section, which is in its own API"

-    assert watch.get('viewed') == False
-    # Loading the most recent snapshot should force viewed to become true
-    client.get(url_for("diff_history_page", uuid="first"), follow_redirects=True)
-
-    # Fetch the whole watch again, viewed should be true
-    res = client.get(
-        url_for("watch", uuid=watch_uuid),
-        headers={'x-api-key': api_key}
-    )
-    watch = res.json
-    assert watch.get('viewed') == True
-
    # basic systeminfo check
    res = client.get(
        url_for("systeminfo"),
@@ -357,25 +343,3 @@ def test_api_watch_PUT_update(client, live_server):
    # Cleanup everything
    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
    assert b'Deleted' in res.data
-
-
-def test_api_import(client, live_server):
-    api_key = extract_api_key_from_UI(client)
-
-    res = client.post(
-        url_for("import") + "?tag=import-test",
-        data='https://website1.com\r\nhttps://website2.com',
-        headers={'x-api-key': api_key},
-        follow_redirects=True
-    )
-
-    assert res.status_code == 200
-    assert len(res.json) == 2
-    res = client.get(url_for("index"))
-    assert b"https://website1.com" in res.data
-    assert b"https://website2.com" in res.data
-
-    # Should see the new tag in the tag/groups list
-    res = client.get(url_for('tags.tags_overview_page'))
-    assert b'import-test' in res.data
-    
@@ -281,8 +281,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):

    # CUSTOM JSON BODY CHECK for POST://
    set_original_response()
-    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}"

    res = client.post(
        url_for("settings_page"),
@@ -298,7 +297,10 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
        follow_redirects=True
    )
    assert b'Settings updated' in res.data
-
+    client.get(
+        url_for("form_delete", uuid="all"),
+        follow_redirects=True
+    )
    # Add a watch and trigger a HTTP POST
    test_url = url_for('test_endpoint', _external=True)
    res = client.post(
@@ -313,9 +315,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
    set_modified_response()

    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(2) # plus extra delay for notifications to fire
+    time.sleep(2)

    with open("test-datastore/notification.txt", 'r') as f:
        x = f.read()
@@ -328,13 +328,6 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
    with open("test-datastore/notification-url.txt", 'r') as f:
        notification_url = f.read()
        assert 'xxx=http' in notification_url
-        # apprise style headers should be stripped
-        assert 'custom-header' not in notification_url
-
-    with open("test-datastore/notification-headers.txt", 'r') as f:
-        notification_headers = f.read()
-        assert 'custom-header: 123' in notification_headers.lower()
-

    # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
    assert os.path.isfile("test-datastore/notification-content-type.txt")
@@ -342,8 +335,3 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
        assert 'application/json' in f.read()

    os.unlink("test-datastore/notification-url.txt")
-
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )
@@ -2,8 +2,9 @@

 import time
 from flask import url_for
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+from .util import set_original_response, set_modified_response, live_server_setup

+sleep_time_for_fetch_thread = 3

 # `subtractive_selectors` should still work in `source:` type requests
 def test_fetch_pdf(client, live_server):
@@ -21,9 +22,7 @@ def test_fetch_pdf(client, live_server):

    assert b"1 Imported" in res.data

-
-    wait_for_all_checks(client)
-
+    time.sleep(sleep_time_for_fetch_thread)
    res = client.get(
        url_for("preview_page", uuid="first"),
        follow_redirects=True
@@ -34,42 +33,8 @@ def test_fetch_pdf(client, live_server):

    # So we know if the file changes in other ways
    import hashlib
-    original_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
+    md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
    # We should have one
-    assert len(original_md5) >0
+    assert len(md5) >0
    # And it's going to be in the document
-    assert b'Document checksum - '+bytes(str(original_md5).encode('utf-8')) in res.data
-
-
-    shutil.copy("tests/test2.pdf", "test-datastore/endpoint-test.pdf")
-    changed_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
-
-    wait_for_all_checks(client)
-
-    # Now something should be ready, indicated by having a 'unviewed' class
-    res = client.get(url_for("index"))
-    assert b'unviewed' in res.data
-
-    # The original checksum should be not be here anymore (cdio adds it to the bottom of the text)
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert original_md5.encode('utf-8') not in res.data
-    assert changed_md5.encode('utf-8') in res.data
-
-
-    res = client.get(
-        url_for("diff_history_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert original_md5.encode('utf-8') in res.data
-    assert changed_md5.encode('utf-8') in res.data
-
-    assert b'here is a change' in res.data
-    
+    assert b'Document checksum - '+bytes(str(md5).encode('utf-8')) in res.data
@@ -205,9 +205,6 @@ def live_server_setup(live_server):
        with open("test-datastore/notification-url.txt", "w") as f:
            f.write(request.url)

-        with open("test-datastore/notification-headers.txt", "w") as f:
-            f.write(str(request.headers))
-
        if request.content_type:
            with open("test-datastore/notification-content-type.txt", "w") as f:
                f.write(request.content_type)
@@ -1,9 +1,13 @@
+import importlib
 import os
+import re
 import threading
 import queue
 import time
+from distutils.util import strtobool

 from changedetectionio import content_fetcher, html_tools
+
 from .processors.text_json_diff import FilterNotFoundInResponse
 from .processors.restock_diff import UnableToExtractRestockData

@@ -15,6 +19,7 @@ from .processors.restock_diff import UnableToExtractRestockData
 import logging
 import sys

+
 class update_worker(threading.Thread):
    current_uuid = None

@@ -24,63 +29,50 @@ class update_worker(threading.Thread):
        self.app = app
        self.notification_q = notification_q
        self.datastore = datastore
+
        super().__init__(*args, **kwargs)

-    def queue_notification_for_watch(self, notification_q, n_object, watch):
+    def queue_notification_for_watch(self, n_object, watch):

        from changedetectionio import diff
-        dates = []
-        trigger_text = ''
-
-        if watch:
-            watch_history = watch.history
-            dates = list(watch_history.keys())
-            trigger_text = watch.get('trigger_text', [])

+        watch_history = watch.history
+        dates = list(watch_history.keys())
        # Add text that was triggered
-        if len(dates):
-            snapshot_contents = watch.get_history_snapshot(dates[-1])
-        else:
-            snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
+        snapshot_contents = watch.get_history_snapshot(dates[-1])

        # HTML needs linebreak, but MarkDown and Text can use a linefeed
-        if n_object.get('notification_format') == 'HTML':
+        if n_object['notification_format'] == 'HTML':
            line_feed_sep = "<br>"
            # Snapshot will be plaintext on the disk, convert to some kind of HTML
            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
        else:
            line_feed_sep = "\n"

+        trigger_text = watch.get('trigger_text', [])
        triggered_text = ''
+
        if len(trigger_text):
            from . import html_tools
            triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
            if triggered_text:
                triggered_text = line_feed_sep.join(triggered_text)

-        # Could be called as a 'test notification' with only 1 snapshot available
-        prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
-        current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
-
-        if len(dates) > 1:
-            prev_snapshot = watch.get_history_snapshot(dates[-2])
-            current_snapshot = watch.get_history_snapshot(dates[-1])

        n_object.update({
            'current_snapshot': snapshot_contents,
-            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep),
-            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
-            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep),
-            'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
-            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
-            'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
+            'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
+            'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
+            'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
+            'diff_patch': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep, patch_format=True),
+            'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
+            'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
            'triggered_text': triggered_text,
-            'uuid': watch.get('uuid') if watch else None,
-            'watch_url': watch.get('url') if watch else None,
+            'uuid': watch.get('uuid'),
+            'watch_url': watch.get('url'),
        })
-        logging.info(">> SENDING NOTIFICATION")
-
-        notification_q.put(n_object)
+        logging.info (">> SENDING NOTIFICATION")
+        self.notification_q.put(n_object)

    # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
    def _check_cascading_vars(self, var_name, watch):
@@ -148,7 +140,7 @@ class update_worker(threading.Thread):
        queued = False
        if n_object and n_object.get('notification_urls'):
            queued = True
-            self.queue_notification_for_watch(notification_q=self.notification_q, n_object=n_object, watch=watch)
+            self.queue_notification_for_watch(n_object, watch)

        return queued

@@ -188,9 +180,9 @@ class update_worker(threading.Thread):
            return
        threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
        n_object = {'notification_title': "Changedetection.io - Alert - Browser step at position {} could not be run".format(step_n+1),
-                    'notification_body': "Your configured browser step at position {} for {{{{watch_url}}}} "
+                    'notification_body': "Your configured browser step at position {} for {{watch['url']}} "
                                         "did not appear on the page after {} attempts, did the page change layout? "
-                                         "Does it need a delay added?\n\nLink: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\n"
+                                         "Does it need a delay added?\n\nLink: {{base_url}}/edit/{{watch_uuid}}\n\n"
                                         "Thanks - Your omniscient changedetection.io installation :)\n".format(step_n+1, threshold),
                    'notification_format': 'text'}

@@ -223,7 +215,7 @@ class update_worker(threading.Thread):
        from .processors import text_json_diff, restock_diff

        while not self.app.config.exit.is_set():
-            update_handler = None
+            change_processor = None

            try:
                queued_item_data = self.q.get(block=False)
@@ -244,35 +236,46 @@ class update_worker(threading.Thread):
                    now = time.time()

                    try:
-                        # Processor is what we are using for detecting the "Change"
+                        # Protect against file:// access
+                        if re.search(r'^file://', self.datastore.data['watching'][uuid].get('url', '').strip(), re.IGNORECASE):
+                            if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
+                                raise Exception(
+                                    "file:// type access is denied for security reasons."
+                                )
+
+                        prefer_fetch_backend = self.datastore.data['watching'][uuid].get('fetch_backend', 'system')
+                        if not prefer_fetch_backend or prefer_fetch_backend == 'system':
+                            prefer_fetch_backend = self.datastore.data['settings']['application'].get('fetch_backend')
+
                        processor = self.datastore.data['watching'][uuid].get('processor', 'text_json_diff')
-                        # if system...

-                        # Abort processing when the content was the same as the last fetch
-                        skip_when_same_checksum = queued_item_data.item.get('skip_when_checksum_same')
+                        processor = 'cdio_whois_diff'

-
-                        # @todo some way to switch by name
-                        # Init a new 'difference_detection_processor'
-
-                        if processor == 'restock_diff':
-                            update_handler = restock_diff.perform_site_check(datastore=self.datastore,
-                                                                             watch_uuid=uuid
-                                                                             )
+                        if processor in ['text_json_diff', 'restock_diff']:
+                            base_processor_module = f"changedetectionio.processors.{processor}"
                        else:
-                            # Used as a default and also by some tests
-                            update_handler = text_json_diff.perform_site_check(datastore=self.datastore,
-                                                                               watch_uuid=uuid
-                                                                               )
+                            # Each plugin is one processor exactly
+                            base_processor_module = f"{processor}.processor"
+
+# its correct that processor dictates which fethcer it uses i think
+
+                        # these should inherit the right fetcher too
+                        module = importlib.import_module(base_processor_module)
+                        change_processor = getattr(module, 'perform_site_check')
+                        change_processor = change_processor(datastore=self.datastore,
+                                                            watch_uuid=uuid,
+                                                            prefer_fetch_backend=prefer_fetch_backend
+                                                            )

                        # Clear last errors (move to preflight func?)
                        self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None

-                        update_handler.call_browser()
-
-                        changed_detected, update_obj, contents = update_handler.run_changedetection(uuid,
-                                                                                    skip_when_checksum_same=skip_when_same_checksum,
-                                                                                    )
+                        skip_when_same_checksum = queued_item_data.item.get('skip_when_checksum_same')
+                        # Each processor extends base class of the kind of fetcher it needs to run anyway
+                        change_processor.fetch_content()
+                        changed_detected, update_obj, contents = change_processor.run_changedetection(uuid,
+                                                                                                      skip_when_checksum_same=skip_when_same_checksum
+                                                                                                      )

                        # Re #342
                        # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
@@ -354,32 +357,20 @@ class update_worker(threading.Thread):
                        changed_detected = False
                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False})

-                    except content_fetcher.BrowserStepsStepException as e:
+                    except content_fetcher.BrowserStepsStepTimout as e:

                        if not self.datastore.data['watching'].get(uuid):
                            continue

                        error_step = e.step_n + 1
-                        from playwright._impl._errors import TimeoutError, Error
-
-                        # Generally enough info for TimeoutError (couldnt locate the element after default seconds)
-                        err_text = f"Browser step at position {error_step} could not run, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step."
-
-                        if e.original_e.name == "TimeoutError":
-                            # Just the first line is enough, the rest is the stack trace
-                            err_text += " Could not find the target."
-                        else:
-                            # Other Error, more info is good.
-                            err_text += " " + str(e.original_e).splitlines()[0]
-
-                        print(f"BrowserSteps exception at step {error_step}", str(e.original_e))
-
+                        err_text = f"Warning, browser step at position {error_step} could not run, target not found, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step"
                        self.datastore.update_watch(uuid=uuid,
                                                    update_obj={'last_error': err_text,
                                                                'browser_steps_last_error_step': error_step
                                                                }
                                                    )

+
                        if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
                            c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
                            c += 1
@@ -491,10 +482,10 @@ class update_worker(threading.Thread):
                                                                           })

                        # Always save the screenshot if it's available
-                        if update_handler.screenshot:
-                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=update_handler.screenshot)
-                        if update_handler.xpath_data:
-                            self.datastore.save_xpath_data(watch_uuid=uuid, data=update_handler.xpath_data)
+                        if change_processor.screenshot:
+                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=change_processor.screenshot)
+                        if change_processor.xpath_data:
+                            self.datastore.save_xpath_data(watch_uuid=uuid, data=change_processor.xpath_data)


                self.current_uuid = None  # Done
@@ -90,7 +90,7 @@ services:
 #

     # Used for fetching pages via Playwright+Chrome where you need Javascript support.
-     # Note: works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector) and other issues
+     # Note: works well but is deprecated, doesnt fetch full page screenshots and other issues
 #    browser-chrome:
 #        hostname: browser-chrome
 #        image: selenium/standalone-chrome:4
@@ -5,13 +5,13 @@
  <meta name="description" content="Manage your changedetection.io watches via API, requires the `x-api-key` header which is found in the settings UI.">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-  <link href="assets/bootstrap.min.css?v=1701595483622" rel="stylesheet" media="screen">
-  <link href="assets/prism.css?v=1701595483622" rel="stylesheet" />
-  <link href="assets/main.css?v=1701595483622" rel="stylesheet" media="screen, print">
-  <link href="assets/favicon.ico?v=1701595483622" rel="icon" type="image/x-icon">
-  <link href="assets/apple-touch-icon.png?v=1701595483622" rel="apple-touch-icon" sizes="180x180">
-  <link href="assets/favicon-32x32.png?v=1701595483622" rel="icon" type="image/png" sizes="32x32">
-  <link href="assets/favicon-16x16.png?v=1701595483622" rel="icon" type="image/png" sizes="16x16">
+  <link href="assets/bootstrap.min.css?v=1677105736053" rel="stylesheet" media="screen">
+  <link href="assets/prism.css?v=1677105736053" rel="stylesheet" />
+  <link href="assets/main.css?v=1677105736053" rel="stylesheet" media="screen, print">
+  <link href="assets/favicon.ico?v=1677105736053" rel="icon" type="image/x-icon">
+  <link href="assets/apple-touch-icon.png?v=1677105736053" rel="apple-touch-icon" sizes="180x180">
+  <link href="assets/favicon-32x32.png?v=1677105736053" rel="icon" type="image/png" sizes="32x32">
+  <link href="assets/favicon-16x16.png?v=1677105736053" rel="icon" type="image/png" sizes="16x16">
 </head>

 <body class="container-fluid">
@@ -928,6 +928,6 @@
  </div>
 </div>

-<script src="assets/main.bundle.js?v=1701595483622"></script>
+<script src="assets/main.bundle.js?v=1677105736053"></script>
 </body>
 </html>
@@ -0,0 +1,2 @@
+run:
+  changedetection: python3 ./changedetection.py -C -d ./datastore -p $PORT
@@ -30,7 +30,7 @@ dnspython~=2.4 # related to eventlet fixes
 # jq not available on Windows so must be installed manually

 # Notification library
-apprise~=1.7.1
+apprise~=1.6.0

 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt
@@ -27,12 +27,12 @@ install_requires = open('requirements.txt').readlines()
 setup(
    name='changedetection.io',
    version=find_version("changedetectionio", "__init__.py"),
-    description='Website change detection and monitoring service, detect changes to web pages and send alerts/notifications.',
+    description='Website change detection and monitoring service',
    long_description=open('README-pip.md').read(),
    long_description_content_type='text/markdown',
    keywords='website change monitor for changes notification change detection '
             'alerts tracking website tracker change alert website and monitoring',
-    entry_points={"console_scripts": ["changedetection.io=changedetectionio:main"]},
+    entry_points={"console_scripts": ["changedetection.io=changedetectionio.changedetection:main"]},
    zip_safe=True,
    scripts=["changedetection.py"],
    author='dgtlmoon',
@@ -41,7 +41,7 @@ setup(
    include_package_data=True,
    install_requires=install_requires,
    license="Apache License 2.0",
-    python_requires=">= 3.10",
+    python_requires=">= 3.7",
    classifiers=['Intended Audience :: Customer Service',
                 'Intended Audience :: Developers',
                 'Intended Audience :: Education',
				`@@ -0,0 +1 @@`
				`web: python3 ./changedetection.py -C -d ./datastore -p $PORT`