Fixing test for CVE-2024-56509

.github/test/Dockerfile-alpine

@@ -2,33 +2,32 @@
 # Test that we can still build on Alpine (musl modified libc https://musl.libc.org/)
 # Some packages wont install via pypi because they dont have a wheel available under this architecture.
 
-FROM ghcr.io/linuxserver/baseimage-alpine:3.21
+FROM ghcr.io/linuxserver/baseimage-alpine:3.18
 ENV PYTHONUNBUFFERED=1
 
 COPY requirements.txt /requirements.txt
 
 RUN \
- apk add --update --no-cache --virtual=build-dependencies \
-    build-base \
+  apk add --update --no-cache --virtual=build-dependencies \
     cargo \
-    git \
+    g++ \
+    gcc \
     jpeg-dev \
     libc-dev \
     libffi-dev \
+    libjpeg \
     libxslt-dev \
+    make \
     openssl-dev \
+    py3-wheel \
     python3-dev \
-    zip \
     zlib-dev && \
   apk add --update --no-cache \
-    libjpeg \
     libxslt \
-    nodejs \
-    poppler-utils \
-    python3 && \
+    python3 \
+    py3-pip && \
   echo "**** pip3 install test of changedetection.io ****" && \
-  python3 -m venv /lsiopy  && \
-  pip install -U pip wheel setuptools && \
-  pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.21/ -r /requirements.txt && \
+  pip3 install -U pip wheel setuptools && \
+  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.18/ -r /requirements.txt && \
   apk del --purge \
     build-dependencies

.github/workflows/containers.yml

@@ -103,19 +103,6 @@ jobs:
 #          provenance: false
 
       # A new tagged release is required, which builds :tag and :latest
-      - name: Docker meta :tag
-        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
-        uses: docker/metadata-action@v5
-        id: meta
-        with:
-            images: |
-                ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
-                ghcr.io/dgtlmoon/changedetection.io
-            tags: |
-                type=semver,pattern={{version}}
-                type=semver,pattern={{major}}.{{minor}}
-                type=semver,pattern={{major}}
-
       - name: Build and push :tag
         id: docker_build_tag_release
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
@@ -124,7 +111,11 @@ jobs:
           context: ./
           file: ./Dockerfile
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
+            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
+            ghcr.io/dgtlmoon/changedetection.io:latest
           platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max

.github/workflows/pypi-release.yml

@@ -45,12 +45,9 @@ jobs:
     - name: Test that the basic pip built package runs without error
       run: |
         set -ex
-        ls -alR 
-        
-        # Find and install the first .whl file
-        find dist -type f -name "*.whl" -exec pip3 install {} \; -quit
+        sudo pip3 install --upgrade pip 
+        pip3 install dist/changedetection.io*.whl
         changedetection.io -d /tmp -p 10000 &
-        
         sleep 3
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/ >/dev/null

.github/workflows/test-stack-reusable-workflow.yml

@@ -64,16 +64,14 @@ jobs:
           echo "Running processes in docker..."
           docker ps
 
-      - name: Run Unit Tests
+      - name: Test built container with Pytest (generally as requests/plaintext fetching)
         run: |
           # Unit tests
+          echo "run test with unittest"
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
-
-      - name: Test built container with Pytest (generally as requests/plaintext fetching)
-        run: |
+          
           # All tests
           echo "run test with pytest"
           # The default pytest logger_level is TRACE

MANIFEST.in

@@ -2,7 +2,6 @@ recursive-include changedetectionio/api *
 recursive-include changedetectionio/apprise_plugin *
 recursive-include changedetectionio/blueprint *
 recursive-include changedetectionio/content_fetchers *
-recursive-include changedetectionio/conditions *
 recursive-include changedetectionio/model *
 recursive-include changedetectionio/processors *
 recursive-include changedetectionio/static *

README.md

@@ -120,7 +120,7 @@ Easily add the current web page to your changedetection.io tool, simply install
 
 [<img src="./docs/chrome-extension-screenshot.png" style="max-width:80%;" alt="Chrome Extension to easily add the current web-page to detect a change."  title="Chrome Extension to easily add the current web-page to detect a change."  />](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
 
-[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop) ( Or check out the [GitHub repo](https://github.com/dgtlmoon/changedetection.io-browser-extension) ) 
+[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
 
 ## Installation
 

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.49.4'
+__version__ = '0.48.04'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
@@ -24,9 +24,6 @@ from loguru import logger
 app = None
 datastore = None
 
-def get_version():
-    return __version__
-
 # Parent wrapper or OS sends us a SIGTERM/SIGINT, do everything required for a clean shutdown
 def sigshutdown_handler(_signo, _stack_frame):
     global app

changedetectionio/api/api_schema.py

@@ -112,35 +112,6 @@ def build_watch_json_schema(d):
 
     schema['properties']['time_between_check'] = build_time_between_check_json_schema()
 
-    schema['properties']['browser_steps'] = {
-        "anyOf": [
-            {
-                "type": "array",
-                "items": {
-                    "type": "object",
-                    "properties": {
-                        "operation": {
-                            "type": ["string", "null"],
-                            "maxLength": 5000  # Allows null and any string up to 5000 chars (including "")
-                        },
-                        "selector": {
-                            "type": ["string", "null"],
-                            "maxLength": 5000
-                        },
-                        "optional_value": {
-                            "type": ["string", "null"],
-                            "maxLength": 5000
-                        }
-                    },
-                    "required": ["operation", "selector", "optional_value"],
-                    "additionalProperties": False  # No extra keys allowed
-                }
-            },
-            {"type": "null"},  # Allows null for `browser_steps`
-            {"type": "array", "maxItems": 0}  # Allows empty array []
-        ]
-    }
-
     # headers ?
     return schema
 

changedetectionio/api/api_v1.py

@@ -76,7 +76,6 @@ class Watch(Resource):
         # Return without history, get that via another API call
         # Properties are not returned as a JSON, so add the required props manually
         watch['history_n'] = watch.history_n
-        # attr .last_changed will check for the last written text snapshot on change
         watch['last_changed'] = watch.last_changed
         watch['viewed'] = watch.viewed
         return watch

changedetectionio/apprise_asset.py

@@ -1,4 +1,3 @@
-from changedetectionio import apprise_plugin
 import apprise
 
 # Create our AppriseAsset and populate it with some of our new values:

changedetectionio/apprise_plugin/__init__.py

@@ -1,8 +1,6 @@
 # include the decorator
 from apprise.decorators import notify
 from loguru import logger
-from requests.structures import CaseInsensitiveDict
-
 
 @notify(on="delete")
 @notify(on="deletes")
@@ -15,84 +13,70 @@ from requests.structures import CaseInsensitiveDict
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     import requests
     import json
-    import re
-
     from urllib.parse import unquote_plus
-    from apprise.utils.parse import parse_url as apprise_parse_url
+    from apprise.utils import parse_url as apprise_parse_url
+    from apprise import URLBase
 
     url = kwargs['meta'].get('url')
-    schema = kwargs['meta'].get('schema').lower().strip()
 
-    # Choose POST, GET etc from requests
-    method =  re.sub(rf's$', '', schema)
-    requests_method = getattr(requests, method)
+    if url.startswith('post'):
+        r = requests.post
+    elif url.startswith('get'):
+        r = requests.get
+    elif url.startswith('put'):
+        r = requests.put
+    elif url.startswith('delete'):
+        r = requests.delete
 
-    params = CaseInsensitiveDict({}) # Added to requests
+    url = url.replace('post://', 'http://')
+    url = url.replace('posts://', 'https://')
+    url = url.replace('put://', 'http://')
+    url = url.replace('puts://', 'https://')
+    url = url.replace('get://', 'http://')
+    url = url.replace('gets://', 'https://')
+    url = url.replace('put://', 'http://')
+    url = url.replace('puts://', 'https://')
+    url = url.replace('delete://', 'http://')
+    url = url.replace('deletes://', 'https://')
+
+    headers = {}
+    params = {}
     auth = None
-    has_error = False
 
     # Convert /foobar?+some-header=hello to proper header dictionary
     results = apprise_parse_url(url)
+    if results:
+        # Add our headers that the user can potentially over-ride if they wish
+        # to to our returned result set and tidy entries by unquoting them
+        headers = {unquote_plus(x): unquote_plus(y)
+                   for x, y in results['qsd+'].items()}
 
-    # Add our headers that the user can potentially over-ride if they wish
-    # to to our returned result set and tidy entries by unquoting them
-    headers = CaseInsensitiveDict({unquote_plus(x): unquote_plus(y)
-               for x, y in results['qsd+'].items()})
+        # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+        # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
+        # but here we are making straight requests, so we need todo convert this against apprise's logic
+        for k, v in results['qsd'].items():
+            if not k.strip('+-') in results['qsd+'].keys():
+                params[unquote_plus(k)] = unquote_plus(v)
 
-    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-    # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
-    # but here we are making straight requests, so we need todo convert this against apprise's logic
-    for k, v in results['qsd'].items():
-        if not k.strip('+-') in results['qsd+'].keys():
-            params[unquote_plus(k)] = unquote_plus(v)
+        # Determine Authentication
+        auth = ''
+        if results.get('user') and results.get('password'):
+            auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
+        elif results.get('user'):
+            auth = (unquote_plus(results.get('user')))
 
-    # Determine Authentication
-    auth = ''
-    if results.get('user') and results.get('password'):
-        auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
-    elif results.get('user'):
-        auth = (unquote_plus(results.get('user')))
-
-    # If it smells like it could be JSON and no content-type was already set, offer a default content type.
-    if body and '{' in body[:100] and not headers.get('Content-Type'):
-        json_header = 'application/json; charset=utf-8'
-        try:
-            # Try if it's JSON
-            json.loads(body)
-            headers['Content-Type'] = json_header
-        except ValueError as e:
-            logger.warning(f"Could not automatically add '{json_header}' header to the notification because the document failed to parse as JSON: {e}")
-            pass
-
-    # POSTS -> HTTPS etc
-    if schema.lower().endswith('s'):
-        url = re.sub(rf'^{schema}', 'https', results.get('url'))
-    else:
-        url = re.sub(rf'^{schema}', 'http', results.get('url'))
-
-    status_str = ''
+    # Try to auto-guess if it's JSON
+    h = 'application/json; charset=utf-8'
     try:
-        r = requests_method(url,
-          auth=auth,
-          data=body.encode('utf-8') if type(body) is str else body,
-          headers=headers,
-          params=params
-        )
+        json.loads(body)
+        headers['Content-Type'] = h
+    except ValueError as e:
+        logger.warning(f"Could not automatically add '{h}' header to the {kwargs['meta'].get('schema')}:// notification because the document failed to parse as JSON: {e}")
+        pass
 
-        if not (200 <= r.status_code < 300):
-            status_str = f"Error sending '{method.upper()}' request to {url} - Status: {r.status_code}: '{r.reason}'"
-            logger.error(status_str)
-            has_error = True
-        else:
-            logger.info(f"Sent '{method.upper()}' request to {url}")
-            has_error = False
-
-    except requests.RequestException as e:
-        status_str = f"Error sending '{method.upper()}' request to {url} - {str(e)}"
-        logger.error(status_str)
-        has_error = True
-
-    if has_error:
-        raise TypeError(status_str)
-
-    return True
+    r(results.get('url'),
+      auth=auth,
+      data=body.encode('utf-8') if type(body) is str else body,
+      headers=headers,
+      params=params
+      )

changedetectionio/blueprint/browser_steps/__init__.py

@@ -22,10 +22,7 @@ from loguru import logger
 
 browsersteps_sessions = {}
 io_interface_context = None
-import json
-import base64
-import hashlib
-from flask import Response
+
 
 def construct_blueprint(datastore: ChangeDetectionStore):
     browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
@@ -88,7 +85,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui(
             playwright_browser=browsersteps_start_session['browser'],
             proxy=proxy,
-            start_url=datastore.data['watching'][watch_uuid].link,
+            start_url=datastore.data['watching'][watch_uuid].get('url'),
             headers=datastore.data['watching'][watch_uuid].get('headers')
         )
 
@@ -163,13 +160,14 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         if not browsersteps_sessions.get(browsersteps_session_id):
             return make_response('No session exists under that ID', 500)
 
-        is_last_step = False
+
         # Actions - step/apply/etc, do the thing and return state
         if request.method == 'POST':
             # @todo - should always be an existing session
             step_operation = request.form.get('operation')
             step_selector = request.form.get('selector')
             step_optional_value = request.form.get('optional_value')
+            step_n = int(request.form.get('step_n'))
             is_last_step = strtobool(request.form.get('is_last_step'))
 
             # @todo try.. accept.. nice errors not popups..
@@ -184,6 +182,16 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 # Try to find something of value to give back to the user
                 return make_response(str(e).splitlines()[0], 401)
 
+            # Get visual selector ready/update its data (also use the current filter info from the page?)
+            # When the last 'apply' button was pressed
+            # @todo this adds overhead because the xpath selection is happening twice
+            u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
+            if is_last_step and u:
+                (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data()
+                watch = datastore.data['watching'].get(uuid)
+                if watch:
+                    watch.save_screenshot(screenshot=screenshot)
+                    watch.save_xpath_data(data=xpath_data)
 
 #        if not this_session.page:
 #            cleanup_playwright_session()
@@ -191,35 +199,31 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         # Screenshots and other info only needed on requesting a step (POST)
         try:
-            (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
-            if is_last_step:
-                watch = datastore.data['watching'].get(uuid)
-                u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
-                if watch and u:
-                    watch.save_screenshot(screenshot=screenshot)
-                    watch.save_xpath_data(data=xpath_data)
-
+            state = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
         except playwright._impl._api_types.Error as e:
             return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
-        except Exception as e:
-            return make_response("Error fetching screenshot and element data - " + str(e), 401)
 
-        # SEND THIS BACK TO THE BROWSER
+        # Use send_file() which is way faster than read/write loop on bytes
+        import json
+        from tempfile import mkstemp
+        from flask import send_file
+        tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
 
-        output = {
-            "screenshot": f"data:image/jpeg;base64,{base64.b64encode(screenshot).decode('ascii')}",
-            "xpath_data": xpath_data,
-            "session_age_start": browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
-            "browser_time_remaining": round(remaining)
-        }
-        json_data = json.dumps(output)
+        output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
+            base64.b64encode(state[0]).decode('ascii')),
+            'xpath_data': state[1],
+            'session_age_start': browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
+            'browser_time_remaining': round(remaining)
+        })
 
-        # Generate an ETag (hash of the response body)
-        etag_hash = hashlib.md5(json_data.encode('utf-8')).hexdigest()
+        with os.fdopen(tmp_fd, 'w') as f:
+            f.write(output)
 
-        # Create the response with ETag
-        response = Response(json_data, mimetype="application/json; charset=UTF-8")
-        response.set_etag(etag_hash)
+        response = make_response(send_file(path_or_file=tmp_file,
+                                           mimetype='application/json; charset=UTF-8',
+                                           etag=True))
+        # No longer needed
+        os.unlink(tmp_file)
 
         return response
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -1,15 +1,14 @@
+#!/usr/bin/env python3
+
 import os
 import time
 import re
 from random import randint
 from loguru import logger
 
-from changedetectionio.content_fetchers.helpers import capture_stitched_together_full_page, SCREENSHOT_SIZE_STITCH_THRESHOLD
 from changedetectionio.content_fetchers.base import manage_user_agent
 from changedetectionio.safe_jinja import render as jinja_render
 
-
-
 # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
 # 0- off, 1- on
 browser_step_ui_config = {'Choose one': '0 0',
@@ -32,7 +31,6 @@ browser_step_ui_config = {'Choose one': '0 0',
 #                          'Extract text and use as filter': '1 0',
                           'Goto site': '0 0',
                           'Goto URL': '0 1',
-                          'Make all child elements visible': '1 0',
                           'Press Enter': '0 0',
                           'Select by label': '1 1',
                           'Scroll down': '0 0',
@@ -40,7 +38,6 @@ browser_step_ui_config = {'Choose one': '0 0',
                           'Wait for seconds': '0 1',
                           'Wait for text': '0 1',
                           'Wait for text in element': '1 1',
-                          'Remove elements': '1 0',
                           #                          'Press Page Down': '0 0',
                           #                          'Press Page Up': '0 0',
                           # weird bug, come back to it later
@@ -55,8 +52,6 @@ class steppable_browser_interface():
     page = None
     start_url = None
 
-    action_timeout = 10 * 1000
-
     def __init__(self, start_url):
         self.start_url = start_url
 
@@ -107,7 +102,7 @@ class steppable_browser_interface():
             return
         elem = self.page.get_by_text(value)
         if elem.count():
-            elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
+            elem.first.click(delay=randint(200, 500), timeout=3000)
 
     def action_click_element_containing_text_if_exists(self, selector=None, value=''):
         logger.debug("Clicking element containing text if exists")
@@ -116,7 +111,7 @@ class steppable_browser_interface():
         elem = self.page.get_by_text(value)
         logger.debug(f"Clicking element containing text - {elem.count()} elements found")
         if elem.count():
-            elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
+            elem.first.click(delay=randint(200, 500), timeout=3000)
         else:
             return
 
@@ -124,7 +119,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
 
-        self.page.fill(selector, value, timeout=self.action_timeout)
+        self.page.fill(selector, value, timeout=10 * 1000)
 
     def action_execute_js(self, selector, value):
         response = self.page.evaluate(value)
@@ -135,7 +130,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
 
-        self.page.click(selector=selector, timeout=self.action_timeout + 20 * 1000, delay=randint(200, 500))
+        self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))
 
     def action_click_element_if_exists(self, selector, value):
         import playwright._impl._errors as _api_types
@@ -143,7 +138,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
         try:
-            self.page.click(selector, timeout=self.action_timeout, delay=randint(200, 500))
+            self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
         except _api_types.TimeoutError as e:
             return
         except _api_types.Error as e:
@@ -190,29 +185,11 @@ class steppable_browser_interface():
         self.page.keyboard.press("PageDown", delay=randint(200, 500))
 
     def action_check_checkbox(self, selector, value):
-        self.page.locator(selector).check(timeout=self.action_timeout)
+        self.page.locator(selector).check(timeout=1000)
 
     def action_uncheck_checkbox(self, selector, value):
-        self.page.locator(selector).uncheck(timeout=self.action_timeout)
+        self.page.locator(selector, timeout=1000).uncheck(timeout=1000)
 
-    def action_remove_elements(self, selector, value):
-        """Removes all elements matching the given selector from the DOM."""
-        self.page.locator(selector).evaluate_all("els => els.forEach(el => el.remove())")
-
-    def action_make_all_child_elements_visible(self, selector, value):
-        """Recursively makes all child elements inside the given selector fully visible."""
-        self.page.locator(selector).locator("*").evaluate_all("""
-            els => els.forEach(el => {
-                el.style.display = 'block';   // Forces it to be displayed
-                el.style.visibility = 'visible';   // Ensures it's not hidden
-                el.style.opacity = '1';   // Fully opaque
-                el.style.position = 'relative';   // Avoids 'absolute' hiding
-                el.style.height = 'auto';   // Expands collapsed elements
-                el.style.width = 'auto';   // Ensures full visibility
-                el.removeAttribute('hidden');   // Removes hidden attribute
-                el.classList.remove('hidden', 'd-none');  // Removes common CSS hidden classes
-            })
-        """)
 
 # Responsible for maintaining a live 'context' with the chrome CDP
 # @todo - how long do contexts live for anyway?
@@ -280,7 +257,6 @@ class browsersteps_live_ui(steppable_browser_interface):
         logger.debug(f"Time to browser setup {time.time()-now:.2f}s")
         self.page.wait_for_timeout(1 * 1000)
 
-
     def mark_as_closed(self):
         logger.debug("Page closed, cleaning up..")
 
@@ -298,30 +274,39 @@ class browsersteps_live_ui(steppable_browser_interface):
         now = time.time()
         self.page.wait_for_timeout(1 * 1000)
 
+        # The actual screenshot
+        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
 
-        full_height = self.page.evaluate("document.documentElement.scrollHeight")
-
-        if full_height >= SCREENSHOT_SIZE_STITCH_THRESHOLD:
-            logger.warning(f"Page full Height: {full_height}px longer than {SCREENSHOT_SIZE_STITCH_THRESHOLD}px, using 'stitched screenshot method'.")
-            screenshot = capture_stitched_together_full_page(self.page)
-        else:
-            screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
-
-        logger.debug(f"Time to get screenshot from browser {time.time() - now:.2f}s")
-
-        now = time.time()
         self.page.evaluate("var include_filters=''")
         # Go find the interactive elements
         # @todo in the future, something smarter that can scan for elements with .click/focus etc event handlers?
         elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
         xpath_element_js = xpath_element_js.replace('%ELEMENTS%', elements)
-
         xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
         # So the JS will find the smallest one first
         xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
-        logger.debug(f"Time to scrape xpath element data in browser {time.time()-now:.2f}s")
-
+        logger.debug(f"Time to complete get_current_state of browser {time.time()-now:.2f}s")
+        # except
         # playwright._impl._api_types.Error: Browser closed.
         # @todo show some countdown timer?
         return (screenshot, xpath_data)
 
+    def request_visualselector_data(self):
+        """
+        Does the same that the playwright operation in content_fetcher does
+        This is used to just bump the VisualSelector data so it' ready to go if they click on the tab
+        @todo refactor and remove duplicate code, add include_filters
+        :param xpath_data:
+        :param screenshot:
+        :param current_include_filters:
+        :return:
+        """
+        import importlib.resources
+        self.page.evaluate("var include_filters=''")
+        xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
+        from changedetectionio.content_fetchers import visualselector_xpath_selectors
+        xpath_element_js = xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
+        xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
+        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
+
+        return (screenshot, xpath_data)

changedetectionio/conditions/__init__.py

@@ -1,135 +0,0 @@
-from flask import Blueprint
-
-from json_logic.builtins import BUILTINS
-
-from .exceptions import EmptyConditionRuleRowNotUsable
-from .pluggy_interface import plugin_manager  # Import the pluggy plugin manager
-from . import default_plugin
-
-# List of all supported JSON Logic operators
-operator_choices = [
-    (None, "Choose one"),
-    (">", "Greater Than"),
-    ("<", "Less Than"),
-    (">=", "Greater Than or Equal To"),
-    ("<=", "Less Than or Equal To"),
-    ("==", "Equals"),
-    ("!=", "Not Equals"),
-    ("in", "Contains"),
-    ("!in", "Does Not Contain"),
-]
-
-# Fields available in the rules
-field_choices = [
-    (None, "Choose one"),
-]
-
-# The data we will feed the JSON Rules to see if it passes the test/conditions or not
-EXECUTE_DATA = {}
-
-
-# Define the extended operations dictionary
-CUSTOM_OPERATIONS = {
-    **BUILTINS,  # Include all standard operators
-}
-
-def filter_complete_rules(ruleset):
-    rules = [
-        rule for rule in ruleset
-        if all(value not in ("", False, "None", None) for value in [rule["operator"], rule["field"], rule["value"]])
-    ]
-    return rules
-
-def convert_to_jsonlogic(logic_operator: str, rule_dict: list):
-    """
-    Convert a structured rule dict into a JSON Logic rule.
-
-    :param rule_dict: Dictionary containing conditions.
-    :return: JSON Logic rule as a dictionary.
-    """
-
-
-    json_logic_conditions = []
-
-    for condition in rule_dict:
-        operator = condition["operator"]
-        field = condition["field"]
-        value = condition["value"]
-
-        if not operator or operator == 'None' or not value or not field:
-            raise EmptyConditionRuleRowNotUsable()
-
-        # Convert value to int/float if possible
-        try:
-            if isinstance(value, str) and "." in value and str != "None":
-                value = float(value)
-            else:
-                value = int(value)
-        except (ValueError, TypeError):
-            pass  # Keep as a string if conversion fails
-
-        # Handle different JSON Logic operators properly
-        if operator == "in":
-            json_logic_conditions.append({"in": [value, {"var": field}]})  # value first
-        elif operator in ("!", "!!", "-"):
-            json_logic_conditions.append({operator: [{"var": field}]})  # Unary operators
-        elif operator in ("min", "max", "cat"):
-            json_logic_conditions.append({operator: value})  # Multi-argument operators
-        else:
-            json_logic_conditions.append({operator: [{"var": field}, value]})  # Standard binary operators
-
-    return {logic_operator: json_logic_conditions} if len(json_logic_conditions) > 1 else json_logic_conditions[0]
-
-
-def execute_ruleset_against_all_plugins(current_watch_uuid: str, application_datastruct, ephemeral_data={} ):
-    """
-    Build our data and options by calling our plugins then pass it to jsonlogic and see if the conditions pass
-
-    :param ruleset: JSON Logic rule dictionary.
-    :param extracted_data: Dictionary containing the facts.   <-- maybe the app struct+uuid
-    :return: Dictionary of plugin results.
-    """
-    from json_logic import jsonLogic
-
-    EXECUTE_DATA = {}
-    result = True
-    
-    ruleset_settings = application_datastruct['watching'].get(current_watch_uuid)
-
-    if ruleset_settings.get("conditions"):
-        logic_operator = "and" if ruleset_settings.get("conditions_match_logic", "ALL") == "ALL" else "or"
-        complete_rules = filter_complete_rules(ruleset_settings['conditions'])
-        if complete_rules:
-            # Give all plugins a chance to update the data dict again (that we will test the conditions against)
-            for plugin in plugin_manager.get_plugins():
-                new_execute_data = plugin.add_data(current_watch_uuid=current_watch_uuid,
-                                                   application_datastruct=application_datastruct,
-                                                   ephemeral_data=ephemeral_data)
-
-                if new_execute_data and isinstance(new_execute_data, dict):
-                    EXECUTE_DATA.update(new_execute_data)
-
-            # Create the ruleset
-            ruleset = convert_to_jsonlogic(logic_operator=logic_operator, rule_dict=complete_rules)
-            
-            # Pass the custom operations dictionary to jsonLogic
-            if not jsonLogic(logic=ruleset, data=EXECUTE_DATA, operations=CUSTOM_OPERATIONS):
-                result = False
-
-    return result
-
-
-# Load plugins dynamically
-for plugin in plugin_manager.get_plugins():
-    new_ops = plugin.register_operators()
-    if isinstance(new_ops, dict):
-        CUSTOM_OPERATIONS.update(new_ops)
-
-    new_operator_choices = plugin.register_operator_choices()
-    if isinstance(new_operator_choices, list):
-        operator_choices.extend(new_operator_choices)
-
-    new_field_choices = plugin.register_field_choices()
-    if isinstance(new_field_choices, list):
-        field_choices.extend(new_field_choices)
-

changedetectionio/conditions/blueprint.py

@@ -1,78 +0,0 @@
-# Flask Blueprint Definition
-import json
-
-from flask import Blueprint
-
-from changedetectionio.conditions import execute_ruleset_against_all_plugins
-
-
-def construct_blueprint(datastore):
-    from changedetectionio.flask_app import login_optionally_required
-
-    conditions_blueprint = Blueprint('conditions', __name__, template_folder="templates")
-
-    @conditions_blueprint.route("/<string:watch_uuid>/verify-condition-single-rule", methods=['POST'])
-    @login_optionally_required
-    def verify_condition_single_rule(watch_uuid):
-        """Verify a single condition rule against the current snapshot"""
-        from changedetectionio.processors.text_json_diff import prepare_filter_prevew
-        from flask import request, jsonify
-        from copy import deepcopy
-
-        ephemeral_data = {}
-
-        # Get the watch data
-        watch = datastore.data['watching'].get(watch_uuid)
-        if not watch:
-            return jsonify({'status': 'error', 'message': 'Watch not found'}), 404
-
-        # First use prepare_filter_prevew to process the form data
-        # This will return text_after_filter which is after all current form settings are applied
-        # Create ephemeral data with the text from the current snapshot
-
-        try:
-            # Call prepare_filter_prevew to get a processed version of the content with current form settings
-            # We'll ignore the returned response and just use the datastore which is modified by the function
-
-            # this should apply all filters etc so then we can run the CONDITIONS against the final output text
-            result = prepare_filter_prevew(datastore=datastore,
-                                           form_data=request.form,
-                                           watch_uuid=watch_uuid)
-
-            ephemeral_data['text'] = result.get('after_filter', '')
-            # Create a temporary watch data structure with this single rule
-            tmp_watch_data = deepcopy(datastore.data['watching'].get(watch_uuid))
-
-            # Override the conditions in the temporary watch
-            rule_json = request.args.get("rule")
-            rule = json.loads(rule_json) if rule_json else None
-            tmp_watch_data['conditions'] = [rule]
-            tmp_watch_data['conditions_match_logic'] = "ALL"  # Single rule, so use ALL
-
-            # Create a temporary application data structure for the rule check
-            temp_app_data = {
-                'watching': {
-                    watch_uuid: tmp_watch_data
-                }
-            }
-
-            # Execute the rule against the current snapshot with form data
-            result = execute_ruleset_against_all_plugins(
-                current_watch_uuid=watch_uuid,
-                application_datastruct=temp_app_data,
-                ephemeral_data=ephemeral_data
-            )
-
-            return jsonify({
-                'status': 'success',
-                'result': result,
-                'message': 'Condition passes' if result else 'Condition does not pass'
-            })
-
-        except Exception as e:
-            return jsonify({
-                'status': 'error',
-                'message': f'Error verifying condition: {str(e)}'
-            }), 500
-
-    return conditions_blueprint

changedetectionio/conditions/default_plugin.py

@@ -1,78 +0,0 @@
-import re
-
-import pluggy
-from price_parser import Price
-from loguru import logger
-
-hookimpl = pluggy.HookimplMarker("changedetectionio_conditions")
-
-
-@hookimpl
-def register_operators():
-    def starts_with(_, text, prefix):
-        return text.lower().strip().startswith(str(prefix).strip().lower())
-
-    def ends_with(_, text, suffix):
-        return text.lower().strip().endswith(str(suffix).strip().lower())
-
-    def length_min(_, text, strlen):
-        return len(text) >= int(strlen)
-
-    def length_max(_, text, strlen):
-        return len(text) <= int(strlen)
-
-    # ✅ Custom function for case-insensitive regex matching
-    def contains_regex(_, text, pattern):
-        """Returns True if `text` contains `pattern` (case-insensitive regex match)."""
-        return bool(re.search(pattern, str(text), re.IGNORECASE))
-
-    # ✅ Custom function for NOT matching case-insensitive regex
-    def not_contains_regex(_, text, pattern):
-        """Returns True if `text` does NOT contain `pattern` (case-insensitive regex match)."""
-        return not bool(re.search(pattern, str(text), re.IGNORECASE))
-
-    return {
-        "!contains_regex": not_contains_regex,
-        "contains_regex": contains_regex,
-        "ends_with": ends_with,
-        "length_max": length_max,
-        "length_min": length_min,
-        "starts_with": starts_with,
-    }
-
-@hookimpl
-def register_operator_choices():
-    return [
-        ("starts_with", "Text Starts With"),
-        ("ends_with", "Text Ends With"),
-        ("length_min", "Length minimum"),
-        ("length_max", "Length maximum"),
-        ("contains_regex", "Text Matches Regex"),
-        ("!contains_regex", "Text Does NOT Match Regex"),
-    ]
-
-@hookimpl
-def register_field_choices():
-    return [
-        ("extracted_number", "Extracted number after 'Filters & Triggers'"),
-#        ("meta_description", "Meta Description"),
-#        ("meta_keywords", "Meta Keywords"),
-        ("page_filtered_text", "Page text after 'Filters & Triggers'"),
-        #("page_title", "Page <title>"), # actual page title <title>
-    ]
-
-@hookimpl
-def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
-
-    res = {}
-    if 'text' in ephemeral_data:
-        res['page_filtered_text'] = ephemeral_data['text']
-
-        # Better to not wrap this in try/except so that the UI can see any errors
-        price = Price.fromstring(ephemeral_data.get('text'))
-        if price and price.amount != None:
-            # This is slightly misleading, it's extracting a PRICE not a Number..
-            res['extracted_number'] = float(price.amount)
-            logger.debug(f"Extracted number result: '{price}' - returning float({res['extracted_number']})")
-
-    return res

changedetectionio/conditions/exceptions.py

@@ -1,6 +0,0 @@
-class EmptyConditionRuleRowNotUsable(Exception):
-    def __init__(self):
-        super().__init__("One of the 'conditions' rulesets is incomplete, cannot run.")
-
-    def __str__(self):
-        return self.args[0]

changedetectionio/conditions/form.py

@@ -1,44 +0,0 @@
-# Condition Rule Form (for each rule row)
-from wtforms import Form, SelectField, StringField, validators
-from wtforms import validators
-
-class ConditionFormRow(Form):
-
-    # ✅ Ensure Plugins Are Loaded BEFORE Importing Choices
-    from changedetectionio.conditions import plugin_manager
-    from changedetectionio.conditions import operator_choices, field_choices
-    field = SelectField(
-        "Field",
-        choices=field_choices,
-        validators=[validators.Optional()]
-    )
-
-    operator = SelectField(
-        "Operator",
-        choices=operator_choices,
-        validators=[validators.Optional()]
-    )
-
-    value = StringField("Value", validators=[validators.Optional()])
-
-    def validate(self, extra_validators=None):
-        # First, run the default validators
-        if not super().validate(extra_validators):
-            return False
-
-        # Custom validation logic
-        # If any of the operator/field/value is set, then they must be all set
-        if any(value not in ("", False, "None", None) for value in [self.operator.data, self.field.data, self.value.data]):
-            if not self.operator.data or self.operator.data == 'None':
-                self.operator.errors.append("Operator is required.")
-                return False
-
-            if not self.field.data or self.field.data == 'None':
-                self.field.errors.append("Field is required.")
-                return False
-
-            if not self.value.data:
-                self.value.errors.append("Value is required.")
-                return False
-
-        return True  # Only return True if all conditions pass

changedetectionio/conditions/pluggy_interface.py

@@ -1,44 +0,0 @@
-import pluggy
-from . import default_plugin  # Import the default plugin
-
-# ✅ Ensure that the namespace in HookspecMarker matches PluginManager
-PLUGIN_NAMESPACE = "changedetectionio_conditions"
-
-hookspec = pluggy.HookspecMarker(PLUGIN_NAMESPACE)
-hookimpl = pluggy.HookimplMarker(PLUGIN_NAMESPACE)
-
-
-class ConditionsSpec:
-    """Hook specifications for extending JSON Logic conditions."""
-
-    @hookspec
-    def register_operators():
-        """Return a dictionary of new JSON Logic operators."""
-        pass
-
-    @hookspec
-    def register_operator_choices():
-        """Return a list of new operator choices."""
-        pass
-
-    @hookspec
-    def register_field_choices():
-        """Return a list of new field choices."""
-        pass
-
-    @hookspec
-    def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
-        """Add to the datadict"""
-        pass
-
-# ✅ Set up Pluggy Plugin Manager
-plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE)
-
-# ✅ Register hookspecs (Ensures they are detected)
-plugin_manager.add_hookspecs(ConditionsSpec)
-
-# ✅ Register built-in plugins manually
-plugin_manager.register(default_plugin, "default_plugin")
-
-# ✅ Discover installed plugins from external packages (if any)
-plugin_manager.load_setuptools_entrypoints(PLUGIN_NAMESPACE)

changedetectionio/content_fetchers/helpers.py

@@ -1,104 +0,0 @@
-
-# Pages with a vertical height longer than this will use the 'stitch together' method.
-
-# - Many GPUs have a max texture size of 16384x16384px (or lower on older devices).
-# - If a page is taller than ~8000–10000px, it risks exceeding GPU memory limits.
-# - This is especially important on headless Chromium, where Playwright may fail to allocate a massive full-page buffer.
-
-
-# The size at which we will switch to stitching method
-SCREENSHOT_SIZE_STITCH_THRESHOLD=8000
-
-from loguru import logger
-
-def capture_stitched_together_full_page(page):
-    import io
-    import os
-    import time
-    from PIL import Image, ImageDraw, ImageFont
-
-    MAX_TOTAL_HEIGHT = SCREENSHOT_SIZE_STITCH_THRESHOLD*4  # Maximum total height for the final image (When in stitch mode)
-    MAX_CHUNK_HEIGHT = 4000  # Height per screenshot chunk
-    WARNING_TEXT_HEIGHT = 20  # Height of the warning text overlay
-
-    # Save the original viewport size
-    original_viewport = page.viewport_size
-    now = time.time()
-
-    try:
-        viewport = page.viewport_size
-        page_height = page.evaluate("document.documentElement.scrollHeight")
-
-        # Limit the total capture height
-        capture_height = min(page_height, MAX_TOTAL_HEIGHT)
-
-        images = []
-        total_captured_height = 0
-
-        for offset in range(0, capture_height, MAX_CHUNK_HEIGHT):
-            # Ensure we do not exceed the total height limit
-            chunk_height = min(MAX_CHUNK_HEIGHT, MAX_TOTAL_HEIGHT - total_captured_height)
-
-            # Adjust viewport size for this chunk
-            page.set_viewport_size({"width": viewport["width"], "height": chunk_height})
-
-            # Scroll to the correct position
-            page.evaluate(f"window.scrollTo(0, {offset})")
-
-            # Capture screenshot chunk
-            screenshot_bytes = page.screenshot(type='jpeg', quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
-            images.append(Image.open(io.BytesIO(screenshot_bytes)))
-
-            total_captured_height += chunk_height
-
-            # Stop if we reached the maximum total height
-            if total_captured_height >= MAX_TOTAL_HEIGHT:
-                break
-
-        # Create the final stitched image
-        stitched_image = Image.new('RGB', (viewport["width"], total_captured_height))
-        y_offset = 0
-
-        # Stitch the screenshot chunks together
-        for img in images:
-            stitched_image.paste(img, (0, y_offset))
-            y_offset += img.height
-
-        logger.debug(f"Screenshot stitched together in {time.time()-now:.2f}s")
-
-        # Overlay warning text if the screenshot was trimmed
-        if page_height > MAX_TOTAL_HEIGHT:
-            draw = ImageDraw.Draw(stitched_image)
-            warning_text = f"WARNING: Screenshot was {page_height}px but trimmed to {MAX_TOTAL_HEIGHT}px because it was too long"
-
-            # Load font (default system font if Arial is unavailable)
-            try:
-                font = ImageFont.truetype("arial.ttf", WARNING_TEXT_HEIGHT)  # Arial (Windows/Mac)
-            except IOError:
-                font = ImageFont.load_default()  # Default font if Arial not found
-
-            # Get text bounding box (correct method for newer Pillow versions)
-            text_bbox = draw.textbbox((0, 0), warning_text, font=font)
-            text_width = text_bbox[2] - text_bbox[0]  # Calculate text width
-            text_height = text_bbox[3] - text_bbox[1]  # Calculate text height
-
-            # Define background rectangle (top of the image)
-            draw.rectangle([(0, 0), (viewport["width"], WARNING_TEXT_HEIGHT)], fill="white")
-
-            # Center text horizontally within the warning area
-            text_x = (viewport["width"] - text_width) // 2
-            text_y = (WARNING_TEXT_HEIGHT - text_height) // 2
-
-            # Draw the warning text in red
-            draw.text((text_x, text_y), warning_text, fill="red", font=font)
-
-        # Save or return the final image
-        output = io.BytesIO()
-        stitched_image.save(output, format="JPEG", quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
-        screenshot = output.getvalue()
-
-    finally:
-        # Restore the original viewport size
-        page.set_viewport_size(original_viewport)
-
-    return screenshot

changedetectionio/content_fetchers/playwright.py

@@ -4,7 +4,6 @@ from urllib.parse import urlparse
 
 from loguru import logger
 
-from changedetectionio.content_fetchers.helpers import capture_stitched_together_full_page, SCREENSHOT_SIZE_STITCH_THRESHOLD
 from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable
 
@@ -90,7 +89,6 @@ class fetcher(Fetcher):
         from playwright.sync_api import sync_playwright
         import playwright._impl._errors
         from changedetectionio.content_fetchers import visualselector_xpath_selectors
-        import time
         self.delete_browser_steps_screenshots()
         response = None
 
@@ -181,7 +179,6 @@ class fetcher(Fetcher):
 
             self.page.wait_for_timeout(extra_wait * 1000)
 
-            now = time.time()
             # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
             if current_include_filters is not None:
                 self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters)))
@@ -193,8 +190,6 @@ class fetcher(Fetcher):
             self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
 
             self.content = self.page.content()
-            logger.debug(f"Time to scrape xpath element data in browser {time.time() - now:.2f}s")
-
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
             # JPEG is better here because the screenshots can be very very large
@@ -204,15 +199,10 @@ class fetcher(Fetcher):
             # acceptable screenshot quality here
             try:
                 # The actual screenshot - this always base64 and needs decoding! horrible! huge CPU usage
-                full_height = self.page.evaluate("document.documentElement.scrollHeight")
-
-                if full_height >= SCREENSHOT_SIZE_STITCH_THRESHOLD:
-                    logger.warning(
-                        f"Page full Height: {full_height}px longer than {SCREENSHOT_SIZE_STITCH_THRESHOLD}px, using 'stitched screenshot method'.")
-                    self.screenshot = capture_stitched_together_full_page(self.page)
-                else:
-                    self.screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
-
+                self.screenshot = self.page.screenshot(type='jpeg',
+                                                       full_page=True,
+                                                       quality=int(os.getenv("SCREENSHOT_QUALITY", 72)),
+                                                       )
             except Exception as e:
                 # It's likely the screenshot was too long/big and something crashed
                 raise ScreenshotUnavailable(url=url, status_code=self.status_code)

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -29,11 +29,8 @@ function isItemInStock() {
         'currently unavailable',
         'dieser artikel ist bald wieder verfügbar',
         'dostępne wkrótce',
-        'en rupture',
         'en rupture de stock',
-        'épuisé',
         'esgotado',
-        'indisponible',
         'indisponível',
         'isn\'t in stock right now',
         'isnt in stock right now',
@@ -55,8 +52,6 @@ function isItemInStock() {
         'niet leverbaar',
         'niet op voorraad',
         'no disponible',
-        'non disponibile',
-        'non disponible',
         'no longer in stock',
         'no tickets available',
         'not available',
@@ -69,10 +64,8 @@ function isItemInStock() {
         'não estamos a aceitar encomendas',
         'out of stock',
         'out-of-stock',
-        'plus disponible',
         'prodotto esaurito',
         'produkt niedostępny',
-        'rupture',
         'sold out',
         'sold-out',
         'stokta yok',

changedetectionio/content_fetchers/res/xpath_element_scraper.js

@@ -41,7 +41,7 @@ const findUpTag = (el) => {
 
     //  Strategy 1: If it's an input, with name, and there's only one, prefer that
     if (el.name !== undefined && el.name.length) {
-        var proposed = el.tagName + "[name=\"" + CSS.escape(el.name) + "\"]";
+        var proposed = el.tagName + "[name=" + el.name + "]";
         var proposed_element = window.document.querySelectorAll(proposed);
         if (proposed_element.length) {
             if (proposed_element.length === 1) {
@@ -102,15 +102,13 @@ function collectVisibleElements(parent, visibleElements) {
     const children = parent.children;
     for (let i = 0; i < children.length; i++) {
         const child = children[i];
-        const computedStyle = window.getComputedStyle(child);
-
         if (
             child.nodeType === Node.ELEMENT_NODE &&
-            computedStyle.display !== 'none' &&
-            computedStyle.visibility !== 'hidden' &&
+            window.getComputedStyle(child).display !== 'none' &&
+            window.getComputedStyle(child).visibility !== 'hidden' &&
             child.offsetWidth >= 0 &&
             child.offsetHeight >= 0 &&
-            computedStyle.contentVisibility !== 'hidden'
+            window.getComputedStyle(child).contentVisibility !== 'hidden'
         ) {
             // If the child is an element and is visible, recursively collect visible elements
             collectVisibleElements(child, visibleElements);
@@ -175,7 +173,6 @@ visibleElementsArray.forEach(function (element) {
 
     // Try to identify any possible currency amounts "Sale: 4000" or "Sale now 3000 Kc", can help with the training.
     const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6)) ) &&  /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|,–)/.test(text) ;
-    const computedStyle = window.getComputedStyle(element);
 
     size_pos.push({
         xpath: xpath_result,
@@ -187,10 +184,10 @@ visibleElementsArray.forEach(function (element) {
         tagName: (element.tagName) ? element.tagName.toLowerCase() : '',
         // tagtype used by Browser Steps
         tagtype: (element.tagName.toLowerCase() === 'input' && element.type) ? element.type.toLowerCase() : '',
-        isClickable: computedStyle.cursor === "pointer",
+        isClickable: window.getComputedStyle(element).cursor === "pointer",
         // Used by the keras trainer
-        fontSize: computedStyle.getPropertyValue('font-size'),
-        fontWeight: computedStyle.getPropertyValue('font-weight'),
+        fontSize: window.getComputedStyle(element).getPropertyValue('font-size'),
+        fontWeight: window.getComputedStyle(element).getPropertyValue('font-weight'),
         hasDigitCurrency: hasDigitCurrency,
         label: label,
     });

changedetectionio/flask_app.py

@@ -598,31 +598,17 @@ def changedetection_app(config=None, datastore_o=None):
 
             if 'notification_title' in request.form and request.form['notification_title'].strip():
                 n_object['notification_title'] = request.form.get('notification_title', '').strip()
-            elif datastore.data['settings']['application'].get('notification_title'):
-                n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
-            else:
-                n_object['notification_title'] = "Test title"
 
             if 'notification_body' in request.form and request.form['notification_body'].strip():
                 n_object['notification_body'] = request.form.get('notification_body', '').strip()
-            elif datastore.data['settings']['application'].get('notification_body'):
-                n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
-            else:
-                n_object['notification_body'] = "Test body"
 
-            n_object['as_async'] = False
             n_object.update(watch.extra_notification_token_values())
-            from .notification import process_notification
-            sent_obj = process_notification(n_object, datastore)
 
+            from . import update_worker
+            new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
+            new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch)
         except Exception as e:
-            e_str = str(e)
-            # Remove this text which is not important and floods the container
-            e_str = e_str.replace(
-                "DEBUG - <class 'apprise.decorators.base.CustomNotifyPlugin.instantiate_plugin.<locals>.CustomNotifyPluginWrapper'>",
-                '')
-
-            return make_response(e_str, 400)
+            return make_response(f"Error: str(e)", 400)
 
         return 'OK - Sent test notifications'
 
@@ -712,63 +698,23 @@ def changedetection_app(config=None, datastore_o=None):
         # Does it use some custom form? does one exist?
         processor_name = datastore.data['watching'][uuid].get('processor', '')
         processor_classes = next((tpl for tpl in find_processors() if tpl[1] == processor_name), None)
-        
-        # If it's not found in traditional processors, check if it's a pluggy plugin
         if not processor_classes:
-            try:
-                from changedetectionio.processors.processor_registry import get_processor_form, _get_plugin_name_map
-                
-                # Get all available plugins for debugging
-                available_plugins = list(_get_plugin_name_map().keys())
-                logger.debug(f"Available processor plugins: {available_plugins}")
-                
-                # Try to get the processor form
-                plugin_form_class = get_processor_form(processor_name)
-                
-                if plugin_form_class:
-                    # Use default text_json_diff_form as parent module for plugins
-                    from changedetectionio.processors.text_json_diff import processor as text_json_diff_processor
-                    form_class = forms.processor_text_json_diff_form
-                    parent_module = get_parent_module(text_json_diff_processor)
-                    
-                    # Skip the normal form loading code path
-                    use_plugin_form = True
-                    logger.debug(f"Successfully loaded form for plugin '{processor_name}'")
-                else:
-                    # Check if the plugin is registered but doesn't have a form
-                    if processor_name in available_plugins:
-                        logger.error(f"Plugin '{processor_name}' is registered but has no form class")
-                        flash(f"Plugin '{processor_name}' is registered but has no form class", 'error')
-                    else:
-                        logger.error(f"Cannot find plugin '{processor_name}'. Available plugins: {available_plugins}")
-                        flash(f"Cannot load the edit form for processor/plugin '{processor_name}', plugin missing?", 'error')
-                    return redirect(url_for('index'))
-            except ImportError as e:
-                logger.error(f"Import error when loading plugin form: {str(e)}")
-                flash(f"Cannot load the edit form for processor/plugin '{processor_name}', plugin system not available?", 'error')
-                return redirect(url_for('index'))
-            except Exception as e:
-                logger.error(f"Unexpected error loading plugin form: {str(e)}")
-                flash(f"Error loading plugin form: {str(e)}", 'error')
-                return redirect(url_for('index'))
-        else:
-            # Traditional processor - continue with normal flow
-            parent_module = get_parent_module(processor_classes[0])
-            use_plugin_form = False
-        
-        # Only follow this path for traditional processors
-        if not use_plugin_form:
-            try:
-                # Get the parent of the "processor.py" go up one, get the form (kinda spaghetti but its reusing existing code)
-                forms_module = importlib.import_module(f"{parent_module.__name__}.forms")
-                # Access the 'processor_settings_form' class from the 'forms' module
-                form_class = getattr(forms_module, 'processor_settings_form')
-            except ModuleNotFoundError as e:
-                # .forms didnt exist
-                form_class = forms.processor_text_json_diff_form
-            except AttributeError as e:
-                # .forms exists but no useful form
-                form_class = forms.processor_text_json_diff_form
+            flash(f"Cannot load the edit form for processor/plugin '{processor_classes[1]}', plugin missing?", 'error')
+            return redirect(url_for('index'))
+
+        parent_module = get_parent_module(processor_classes[0])
+
+        try:
+            # Get the parent of the "processor.py" go up one, get the form (kinda spaghetti but its reusing existing code)
+            forms_module = importlib.import_module(f"{parent_module.__name__}.forms")
+            # Access the 'processor_settings_form' class from the 'forms' module
+            form_class = getattr(forms_module, 'processor_settings_form')
+        except ModuleNotFoundError as e:
+            # .forms didnt exist
+            form_class = forms.processor_text_json_diff_form
+        except AttributeError as e:
+            # .forms exists but no useful form
+            form_class = forms.processor_text_json_diff_form
 
         form = form_class(formdata=request.form if request.method == 'POST' else None,
                           data=default,
@@ -915,14 +861,14 @@ def changedetection_app(config=None, datastore_o=None):
 
             system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
 
-            watch_uses_webdriver = False
+            is_html_webdriver = False
             if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
-                watch_uses_webdriver = True
+                is_html_webdriver = True
 
             from zoneinfo import available_timezones
 
             # Only works reliably with Playwright
-
+            visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver
             template_args = {
                 'available_processors': processors.available_processors(),
                 'available_timezones': sorted(available_timezones()),
@@ -935,13 +881,14 @@ def changedetection_app(config=None, datastore_o=None):
                 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False,
                 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
                 'has_special_tag_options': _watch_has_tag_options_set(watch=watch),
-                'watch_uses_webdriver': watch_uses_webdriver,
+                'is_html_webdriver': is_html_webdriver,
                 'jq_support': jq_support,
                 'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False),
                 'settings_application': datastore.data['settings']['application'],
                 'timezone_default_config': datastore.data['settings']['application'].get('timezone'),
                 'using_global_webdriver_wait': not default['webdriver_delay'],
                 'uuid': uuid,
+                'visualselector_enabled': visualselector_enabled,
                 'watch': watch
             }
 
@@ -1422,10 +1369,8 @@ def changedetection_app(config=None, datastore_o=None):
     @login_optionally_required
     def watch_get_preview_rendered(uuid):
         '''For when viewing the "preview" of the rendered text from inside of Edit'''
-        from flask import jsonify
         from .processors.text_json_diff import prepare_filter_prevew
-        result = prepare_filter_prevew(watch_uuid=uuid, form_data=request.form, datastore=datastore)
-        return jsonify(result)
+        return prepare_filter_prevew(watch_uuid=uuid, datastore=datastore)
 
 
     @app.route("/form/add/quickwatch", methods=['POST'])
@@ -1726,9 +1671,6 @@ def changedetection_app(config=None, datastore_o=None):
     import changedetectionio.blueprint.backups as backups
     app.register_blueprint(backups.construct_blueprint(datastore), url_prefix='/backups')
 
-    import changedetectionio.conditions.blueprint as conditions
-    app.register_blueprint(conditions.construct_blueprint(datastore), url_prefix='/conditions')
-
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
     threading.Thread(target=notification_runner).start()

changedetectionio/forms.py

@@ -3,7 +3,6 @@ import re
 from loguru import logger
 from wtforms.widgets.core import TimeInput
 
-from changedetectionio.conditions.form import ConditionFormRow
 from changedetectionio.strtobool import strtobool
 
 from wtforms import (
@@ -172,7 +171,7 @@ class validateTimeZoneName(object):
 
 class ScheduleLimitDaySubForm(Form):
     enabled = BooleanField("not set", default=True)
-    start_time = TimeStringField("Start At", default="00:00", validators=[validators.Optional()])
+    start_time = TimeStringField("Start At", default="00:00", render_kw={"placeholder": "HH:MM"}, validators=[validators.Optional()])
     duration = FormField(TimeDurationForm, label="Run duration")
 
 class ScheduleLimitForm(Form):
@@ -306,10 +305,8 @@ class ValidateAppRiseServers(object):
     def __call__(self, form, field):
         import apprise
         apobj = apprise.Apprise()
-
         # so that the custom endpoints are registered
-        from .apprise_asset import asset
-
+        from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper
         for server_url in field.data:
             url = server_url.strip()
             if url.startswith("#"):
@@ -512,7 +509,6 @@ class quickWatchForm(Form):
     edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
 
 
-
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
     from . import processors
@@ -600,10 +596,6 @@ class processor_text_json_diff_form(commonSettingsForm):
     notification_muted = BooleanField('Notifications Muted / Off', default=False)
     notification_screenshot = BooleanField('Attach screenshot to notification (where possible)', default=False)
 
-    conditions_match_logic = RadioField(u'Match', choices=[('ALL', 'Match all of the following'),('ANY', 'Match any of the following')], default='ALL')
-    conditions = FieldList(FormField(ConditionFormRow), min_entries=1)  # Add rule logic here
-
-
     def extra_tab_content(self):
         return None
 

changedetectionio/html_tools.py

@@ -1,6 +1,5 @@
-from loguru import logger
-from lxml import etree
 from typing import List
+from lxml import etree
 import json
 import re
 
@@ -299,10 +298,8 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 # https://github.com/dgtlmoon/changedetection.io/pull/2041#issuecomment-1848397161w
     # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded within HTML tags
     try:
-        # .lstrip("\ufeff") strings ByteOrderMark from UTF8 and still lets the UTF work
-        stripped_text_from_html = _parse_json(json.loads(content.lstrip("\ufeff") ), json_filter)
-    except json.JSONDecodeError as e:
-        logger.warning(str(e))
+        stripped_text_from_html = _parse_json(json.loads(content), json_filter)
+    except json.JSONDecodeError:
 
         # Foreach <script json></script> blob.. just return the first that matches json_filter
         # As a last resort, try to parse the whole <body>

changedetectionio/model/App.py

@@ -69,7 +69,7 @@ def parse_headers_from_text_file(filepath):
         for l in f.readlines():
             l = l.strip()
             if not l.startswith('#') and ':' in l:
-                (k, v) = l.split(':', 1)  # Split only on the first colon
+                (k, v) = l.split(':')
                 headers[k.strip()] = v.strip()
 
     return headers

changedetectionio/model/Watch.py

@@ -67,6 +67,7 @@ class model(watch_base):
 
     @property
     def link(self):
+
         url = self.get('url', '')
         if not is_safe_url(url):
             return 'DISABLED'
@@ -92,19 +93,6 @@ class model(watch_base):
         # Also double check it after any Jinja2 formatting just incase
         if not is_safe_url(ready_url):
             return 'DISABLED'
-            
-        # Check if a processor wants to customize the display link
-        processor_name = self.get('processor')
-        if processor_name:
-            try:
-                # Import here to avoid circular imports
-                from changedetectionio.processors.processor_registry import get_display_link
-                custom_link = get_display_link(url=ready_url, processor_name=processor_name)
-                if custom_link:
-                    return custom_link
-            except Exception as e:
-                logger.error(f"Error getting custom display link for processor {processor_name}: {str(e)}")
-                
         return ready_url
 
     def clear_watch(self):
@@ -259,32 +247,37 @@ class model(watch_base):
         bump = self.history
         return self.__newest_history_key
 
-    # Given an arbitrary timestamp, find the best history key for the [diff] button so it can preset a smarter from_version
+    # Given an arbitrary timestamp, find the closest next key
+    # For example, last_viewed = 1000 so it should return the next 1001 timestamp
+    #
+    # used for the [diff] button so it can preset a smarter from_version
     @property
-    def get_from_version_based_on_last_viewed(self):
+    def get_next_snapshot_key_to_last_viewed(self):
 
         """Unfortunately for now timestamp is stored as string key"""
         keys = list(self.history.keys())
         if not keys:
             return None
-        if len(keys) == 1:
-            return keys[0]
 
         last_viewed = int(self.get('last_viewed'))
+        prev_k = keys[0]
         sorted_keys = sorted(keys, key=lambda x: int(x))
         sorted_keys.reverse()
 
-        # When the 'last viewed' timestamp is greater than or equal the newest snapshot, return second newest
-        if last_viewed >= int(sorted_keys[0]):
+        # When the 'last viewed' timestamp is greater than the newest snapshot, return second last
+        if last_viewed > int(sorted_keys[0]):
             return sorted_keys[1]
-        
-        # When the 'last viewed' timestamp is between snapshots, return the older snapshot
-        for newer, older in list(zip(sorted_keys[0:], sorted_keys[1:])):
-            if last_viewed < int(newer) and last_viewed >= int(older):
-                return older
 
-        # When the 'last viewed' timestamp is less than the oldest snapshot, return oldest
-        return sorted_keys[-1]
+        for k in sorted_keys:
+            if int(k) < last_viewed:
+                if prev_k == sorted_keys[0]:
+                    # Return the second last one so we dont recommend the same version compares itself
+                    return sorted_keys[1]
+
+                return prev_k
+            prev_k = k
+
+        return keys[0]
 
     def get_history_snapshot(self, timestamp):
         import brotli
@@ -364,7 +357,7 @@ class model(watch_base):
     # Iterate over all history texts and see if something new exists
     # Always applying .strip() to start/end but optionally replace any other whitespace
     def lines_contain_something_unique_compared_to_history(self, lines: list, ignore_whitespace=False):
-        local_lines = set([])
+        local_lines = []
         if lines:
             if ignore_whitespace:
                 if isinstance(lines[0], str): # Can be either str or bytes depending on what was on the disk
@@ -539,7 +532,7 @@ class model(watch_base):
     def save_error_text(self, contents):
         self.ensure_data_dir_exists()
         target_path = os.path.join(self.watch_data_dir, "last-error.txt")
-        with open(target_path, 'w', encoding='utf-8') as f:
+        with open(target_path, 'w') as f:
             f.write(contents)
 
     def save_xpath_data(self, data, as_error=False):

changedetectionio/notification.py

@@ -67,10 +67,6 @@ def process_notification(n_object, datastore):
 
     sent_objs = []
     from .apprise_asset import asset
-
-    if 'as_async' in n_object:
-        asset.async_mode = n_object.get('as_async')
-
     apobj = apprise.Apprise(debug=True, asset=asset)
 
     if not n_object.get('notification_urls'):
@@ -161,6 +157,8 @@ def process_notification(n_object, datastore):
             attach=n_object.get('screenshot', None)
         )
 
+        # Give apprise time to register an error
+        time.sleep(3)
 
         # Returns empty string if nothing found, multi-line string otherwise
         log_value = logs.getvalue()

changedetectionio/processors/__init__.py

@@ -3,7 +3,6 @@ from changedetectionio.content_fetchers.base import Fetcher
 from changedetectionio.strtobool import strtobool
 from copy import deepcopy
 from loguru import logger
-
 import hashlib
 import importlib
 import inspect
@@ -11,10 +10,6 @@ import os
 import pkgutil
 import re
 
-# Import the plugin manager
-from .pluggy_interface import plugin_manager
-
-
 class difference_detection_processor():
 
     browser_steps = None
@@ -31,95 +26,9 @@ class difference_detection_processor():
         self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
         # Generic fetcher that should be extended (requests, playwright etc)
         self.fetcher = Fetcher()
-        
-    def _get_proxy_for_watch(self, preferred_proxy_id=None):
-        """Get proxy configuration based on watch settings and preferred proxy ID
-        
-        Args:
-            preferred_proxy_id: Optional explicit proxy ID to use
-            
-        Returns:
-            dict: Proxy configuration or None if no proxy should be used
-            str: Proxy URL or None if no proxy should be used
-        """
-        # Default to no proxy config
-        proxy_config = None
-        proxy_url = None
-        
-        # Check if datastore is available and has get_preferred_proxy_for_watch method
-        if hasattr(self, 'datastore') and self.datastore:
-            try:
-                # Get preferred proxy ID if not provided
-                if not preferred_proxy_id and hasattr(self.datastore, 'get_preferred_proxy_for_watch'):
-                    # Get the watch UUID if available
-                    watch_uuid = None
-                    if hasattr(self.watch, 'get'):
-                        watch_uuid = self.watch.get('uuid')
-                    elif hasattr(self.watch, 'uuid'):
-                        watch_uuid = self.watch.uuid
-                    
-                    if watch_uuid:
-                        preferred_proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
-                
-                # Check if we have a proxy list and a valid proxy ID
-                if preferred_proxy_id and hasattr(self.datastore, 'proxy_list') and self.datastore.proxy_list:
-                    proxy_info = self.datastore.proxy_list.get(preferred_proxy_id)
-                    
-                    if proxy_info and 'url' in proxy_info:
-                        proxy_url = proxy_info.get('url')
-                        logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}'")
-                        
-                        # Parse the proxy URL to build a proxy dict for requests
-                        import urllib.parse
-                        parsed_proxy = urllib.parse.urlparse(proxy_url)
-                        proxy_type = parsed_proxy.scheme
-                        
-                        # Extract credentials if present
-                        username = None
-                        password = None
-                        if parsed_proxy.username:
-                            username = parsed_proxy.username
-                            if parsed_proxy.password:
-                                password = parsed_proxy.password
-                        
-                        # Build the proxy URL without credentials for the proxy dict
-                        netloc = parsed_proxy.netloc
-                        if '@' in netloc:
-                            netloc = netloc.split('@')[1]
-                        
-                        proxy_addr = f"{proxy_type}://{netloc}"
-                        
-                        # Create the proxy configuration
-                        proxy_config = {
-                            'http': proxy_addr,
-                            'https': proxy_addr
-                        }
-                        
-                        # Add credentials if present
-                        if username:
-                            proxy_config['username'] = username
-                            if password:
-                                proxy_config['password'] = password
-            except Exception as e:
-                # Log the error but continue without a proxy
-                logger.error(f"Error setting up proxy: {str(e)}")
-                proxy_config = None
-                proxy_url = None
-                
-        return proxy_config, proxy_url
 
     def call_browser(self, preferred_proxy_id=None):
-        """Fetch content using the appropriate browser/fetcher
-        
-        This method will:
-        1. Determine the appropriate fetcher to use based on watch settings
-        2. Set up proxy configuration if needed
-        3. Initialize the fetcher with the correct parameters
-        4. Configure any browser steps if needed
-        
-        Args:
-            preferred_proxy_id: Optional explicit proxy ID to use
-        """
+
         from requests.structures import CaseInsensitiveDict
 
         url = self.watch.link
@@ -134,8 +43,8 @@ class difference_detection_processor():
         # Requests, playwright, other browser via wss:// etc, fetch_extra_something
         prefer_fetch_backend = self.watch.get('fetch_backend', 'system')
 
-        # Get proxy configuration
-        proxy_config, proxy_url = self._get_proxy_for_watch(preferred_proxy_id)
+        # Proxy ID "key"
+        preferred_proxy_id = preferred_proxy_id if preferred_proxy_id else self.datastore.get_preferred_proxy_for_watch(uuid=self.watch.get('uuid'))
 
         # Pluggable content self.fetcher
         if not prefer_fetch_backend or prefer_fetch_backend == 'system':
@@ -173,10 +82,14 @@ class difference_detection_processor():
             # What it referenced doesnt exist, Just use a default
             fetcher_obj = getattr(content_fetchers, "html_requests")
 
-        # Custom browser endpoints should NOT have a proxy added
-        if proxy_url and prefer_fetch_backend.startswith('extra_browser_'):
-            logger.debug(f"Skipping adding proxy data when custom Browser endpoint is specified.")
-            proxy_url = None
+        proxy_url = None
+        if preferred_proxy_id:
+            # Custom browser endpoints should NOT have a proxy added
+            if not prefer_fetch_backend.startswith('extra_browser_'):
+                proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')
+                logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}' for {url}")
+            else:
+                logger.debug(f"Skipping adding proxy data when custom Browser endpoint is specified. ")
 
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
@@ -272,9 +185,9 @@ def find_sub_packages(package_name):
 
 def find_processors():
     """
-    Find all subclasses of DifferenceDetectionProcessor in the specified package
-    and also include processors from the plugin system.
+    Find all subclasses of DifferenceDetectionProcessor in the specified package.
 
+    :param package_name: The name of the package to scan for processor modules.
     :return: A list of (module, class) tuples.
     """
     package_name = "changedetectionio.processors"  # Name of the current package/module
@@ -282,7 +195,6 @@ def find_processors():
     processors = []
     sub_packages = find_sub_packages(package_name)
 
-    # Find traditional processors
     for sub_package in sub_packages:
         module_name = f"{package_name}.{sub_package}.processor"
         try:
@@ -295,15 +207,6 @@ def find_processors():
         except (ModuleNotFoundError, ImportError) as e:
             logger.warning(f"Failed to import module {module_name}: {e} (find_processors())")
 
-    # Also include processors from the plugin system
-    try:
-        from .processor_registry import get_plugin_processor_modules
-        plugin_modules = get_plugin_processor_modules()
-        if plugin_modules:
-            processors.extend(plugin_modules)
-    except (ImportError, ModuleNotFoundError) as e:
-        logger.warning(f"Failed to import plugin modules: {e} (find_processors())")
-
     return processors
 
 
@@ -320,22 +223,8 @@ def get_parent_module(module):
     return False
 
 
-def get_custom_watch_obj_for_processor(processor_name):
-    """
-    Get the custom watch object for a processor
-    :param processor_name: Name of the processor
-    :return: Watch class or None
-    """
-    # First, try to get the watch model from the pluggy system
-    try:
-        from .processor_registry import get_processor_watch_model
-        watch_model = get_processor_watch_model(processor_name)
-        if watch_model:
-            return watch_model
-    except Exception as e:
-        logger.warning(f"Error getting processor watch model from pluggy: {e}")
 
-    # Fall back to the traditional approach
+def get_custom_watch_obj_for_processor(processor_name):
     from changedetectionio.model import Watch
     watch_class = Watch.model
     processor_classes = find_processors()
@@ -352,47 +241,14 @@ def get_custom_watch_obj_for_processor(processor_name):
 def available_processors():
     """
     Get a list of processors by name and description for the UI elements
-    :return: A list of tuples (processor_name, description)
+    :return: A list :)
     """
-    # Get processors from the pluggy system
-    pluggy_processors = []
-    try:
-        from .processor_registry import get_all_processors
-        pluggy_processors = get_all_processors()
-    except Exception as e:
-        logger.error(f"Error getting processors from pluggy: {str(e)}")
-    
-    # Get processors from the traditional file-based system
-    traditional_processors = []
-    try:
-        # Let's not use find_processors() directly since it now also includes pluggy processors
-        package_name = "changedetectionio.processors"
-        sub_packages = find_sub_packages(package_name)
-        
-        for sub_package in sub_packages:
-            module_name = f"{package_name}.{sub_package}.processor"
-            try:
-                module = importlib.import_module(module_name)
-                # Get the name and description from the module if available
-                name = getattr(module, 'name', f"Traditional processor: {sub_package}")
-                description = getattr(module, 'description', sub_package)
-                traditional_processors.append((sub_package, name))
-            except (ModuleNotFoundError, ImportError, AttributeError) as e:
-                logger.warning(f"Failed to import module {module_name}: {e} (available_processors())")
-    except Exception as e:
-        logger.error(f"Error getting traditional processors: {str(e)}")
-    
-    # Combine the lists, ensuring no duplicates
-    # Pluggy processors take precedence
-    all_processors = []
-    
-    # Add all pluggy processors
-    all_processors.extend(pluggy_processors)
-    
-    # Add traditional processors that aren't already registered via pluggy
-    pluggy_processor_names = [name for name, _ in pluggy_processors]
-    for processor_class, name in traditional_processors:
-        if processor_class not in pluggy_processor_names:
-            all_processors.append((processor_class, name))
-    
-    return all_processors
+
+    processor_classes = find_processors()
+
+    available = []
+    for package, processor_class in processor_classes:
+        available.append((processor_class, package.name))
+
+    return available
+

changedetectionio/processors/form.py

@@ -1,17 +0,0 @@
-from wtforms import (
-    BooleanField,
-    validators,
-    RadioField
-)
-from wtforms.fields.choices import SelectField
-from wtforms.fields.form import FormField
-from wtforms.form import Form
-
-class BaseProcessorForm(Form):
-    """Base class for processor forms"""
-    
-    def extra_tab_content(self):
-        return None
-
-    def extra_form_content(self):
-        return None

changedetectionio/processors/forms.py

@@ -1,4 +0,0 @@
-"""
-Forms for processors
-"""
-from changedetectionio.forms import processor_text_json_diff_form

changedetectionio/processors/pluggy_interface.py

@@ -1,69 +0,0 @@
-import pluggy
-
-# Define the plugin namespace for processors
-PLUGIN_NAMESPACE = "changedetectionio_processors"
-
-hookspec = pluggy.HookspecMarker(PLUGIN_NAMESPACE)
-hookimpl = pluggy.HookimplMarker(PLUGIN_NAMESPACE)
-
-
-class ProcessorSpec:
-    """Hook specifications for processor plugins."""
-
-    @hookspec
-    def get_processor_name():
-        """Return the name of the processor."""
-        pass
-
-    @hookspec
-    def get_processor_description():
-        """Return the description of the processor."""
-        pass
-    
-    @hookspec
-    def get_processor_class():
-        """Return the processor class."""
-        pass
-    
-    @hookspec
-    def get_processor_form():
-        """Return the processor form class."""
-        pass
-    
-    @hookspec
-    def get_processor_watch_model():
-        """Return the watch model class for this processor (if any)."""
-        pass
-    
-    @hookspec
-    def get_display_link(url, processor_name):
-        """Return a custom display link for the given processor.
-        
-        Args:
-            url: The original URL from the watch
-            processor_name: The name of the processor
-            
-        Returns:
-            A string with the custom display link or None to use the default
-        """
-        pass
-    
-    @hookspec
-    def perform_site_check(datastore, watch_uuid):
-        """Create and return a processor instance ready to perform site check.
-        
-        Args:
-            datastore: The application datastore
-            watch_uuid: The UUID of the watch to check
-            
-        Returns:
-            A processor instance ready to perform site check
-        """
-        pass
-
-
-# Set up the plugin manager
-plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE)
-
-# Register hook specifications
-plugin_manager.add_hookspecs(ProcessorSpec)

changedetectionio/processors/processor_registry.py

@@ -1,222 +0,0 @@
-from loguru import logger
-from changedetectionio.model import Watch
-from .pluggy_interface import plugin_manager
-from typing import Dict, Any, List, Tuple, Optional, TypeVar, Type
-import functools
-
-# Import and register internal plugins
-from . import whois_plugin
-from . import test_plugin
-
-# Register plugins
-plugin_manager.register(whois_plugin)
-plugin_manager.register(test_plugin)
-
-# Load any setuptools entrypoints
-plugin_manager.load_setuptools_entrypoints("changedetectionio_processors")
-
-# Type definitions for better type hinting
-T = TypeVar('T')
-ProcessorClass = TypeVar('ProcessorClass')
-ProcessorForm = TypeVar('ProcessorForm')
-ProcessorWatchModel = TypeVar('ProcessorWatchModel')
-ProcessorInstance = TypeVar('ProcessorInstance')
-
-# Cache for plugin name mapping to improve performance
-_plugin_name_map: Dict[str, Any] = {}
-
-def register_plugin(plugin_module):
-    """Register a processor plugin"""
-    plugin_manager.register(plugin_module)
-    # Clear the plugin name map cache when a new plugin is registered
-    global _plugin_name_map
-    _plugin_name_map = {}
-
-def _get_plugin_name_map() -> Dict[str, Any]:
-    """Get a mapping of processor names to plugins
-    :return: Dictionary mapping processor names to plugins
-    """
-    global _plugin_name_map
-    
-    # Return cached map if available
-    if _plugin_name_map:
-        return _plugin_name_map
-    
-    # Build the map
-    result = {}
-    
-    # Get all plugins from the plugin manager
-    all_plugins = list(plugin_manager.get_plugins())
-    
-    # First register known internal plugins by name for reliability
-    known_plugins = {
-        'whois': whois_plugin,
-        'test': test_plugin
-    }
-    
-    for name, plugin in known_plugins.items():
-        if plugin in all_plugins:
-            result[name] = plugin
-    
-    # Then process remaining plugins through the hook system
-    for plugin in all_plugins:
-        if plugin in known_plugins.values():
-            continue  # Skip plugins we've already registered
-            
-        try:
-            # Get the processor name from this plugin
-            name_results = plugin_manager.hook.get_processor_name(plugin=plugin)
-            
-            if name_results:
-                plugin_name = name_results[0]
-                
-                # Check for name collisions
-                if plugin_name in result:
-                    logger.warning(f"Plugin name collision: '{plugin_name}' is already registered")
-                    continue
-                    
-                result[plugin_name] = plugin
-        except Exception as e:
-            logger.error(f"Error getting processor name from plugin: {str(e)}")
-    
-    # Cache the map
-    _plugin_name_map = result
-    return result
-
-def _get_plugin_by_name(processor_name: str) -> Optional[Any]:
-    """Get a plugin by its processor name
-    :param processor_name: Name of the processor
-    :return: Plugin object or None
-    """
-    return _get_plugin_name_map().get(processor_name)
-
-def _call_hook_for_plugin(plugin: Any, hook_name: str, default_value: T = None, **kwargs) -> Optional[T]:
-    """Call a hook for a specific plugin and handle exceptions
-    :param plugin: The plugin to call the hook for
-    :param hook_name: Name of the hook to call
-    :param default_value: Default value to return if the hook call fails
-    :param kwargs: Additional arguments to pass to the hook
-    :return: Result of the hook call or default value
-    """
-    if not plugin:
-        return default_value
-    
-    try:
-        hook = getattr(plugin_manager.hook, hook_name)
-        results = hook(plugin=plugin, **kwargs)
-        
-        if results:
-            return results[0]
-    except Exception as e:
-        logger.error(f"Error calling {hook_name} for plugin: {str(e)}")
-    
-    return default_value
-
-def get_all_processors() -> List[Tuple[str, str]]:
-    """Get all processors
-    :return: List of tuples (processor_name, processor_description)
-    """
-    processors = []
-    
-    for processor_name, plugin in _get_plugin_name_map().items():
-        description = _call_hook_for_plugin(plugin, 'get_processor_description')
-        if description:
-            processors.append((processor_name, description))
-    
-    return processors
-
-def get_processor_class(processor_name: str) -> Optional[Type[ProcessorClass]]:
-    """Get processor class by name
-    :param processor_name: Name of the processor
-    :return: Processor class or None
-    """
-    plugin = _get_plugin_by_name(processor_name)
-    return _call_hook_for_plugin(plugin, 'get_processor_class')
-
-def get_processor_form(processor_name: str) -> Optional[Type[ProcessorForm]]:
-    """Get processor form by name
-    :param processor_name: Name of the processor
-    :return: Processor form class or None
-    """
-    plugin = _get_plugin_by_name(processor_name)
-    return _call_hook_for_plugin(plugin, 'get_processor_form')
-
-def get_processor_watch_model(processor_name: str) -> Type[ProcessorWatchModel]:
-    """Get processor watch model by name
-    :param processor_name: Name of the processor
-    :return: Watch model class or default Watch model
-    """
-    plugin = _get_plugin_by_name(processor_name)
-    return _call_hook_for_plugin(plugin, 'get_processor_watch_model', default_value=Watch.model)
-
-def get_processor_site_check(processor_name: str, datastore: Any, watch_uuid: str) -> Optional[ProcessorInstance]:
-    """Get a processor instance ready to perform site check
-    :param processor_name: Name of the processor
-    :param datastore: The application datastore
-    :param watch_uuid: The UUID of the watch to check
-    :return: A processor instance ready to perform site check, or None
-    """
-    plugin = _get_plugin_by_name(processor_name)
-    if not plugin:
-        return None
-    
-    # Try to get the perform_site_check implementation
-    try:
-        processor = _call_hook_for_plugin(
-            plugin, 
-            'perform_site_check', 
-            datastore=datastore, 
-            watch_uuid=watch_uuid
-        )
-        if processor:
-            return processor
-        
-        # If no perform_site_check hook implementation, try getting the class and instantiating it
-        processor_class = _call_hook_for_plugin(plugin, 'get_processor_class')
-        if processor_class:
-            return processor_class(datastore=datastore, watch_uuid=watch_uuid)
-    except Exception as e:
-        logger.error(f"Error getting processor site check for {processor_name}: {str(e)}")
-    
-    return None
-
-def get_display_link(url: str, processor_name: str) -> Optional[str]:
-    """Get a custom display link for the given processor
-    :param url: The original URL from the watch
-    :param processor_name: Name of the processor
-    :return: A string with the custom display link or None to use the default
-    """
-    plugin = _get_plugin_by_name(processor_name)
-    return _call_hook_for_plugin(
-        plugin, 
-        'get_display_link', 
-        url=url, 
-        processor_name=processor_name
-    )
-
-def get_plugin_processor_modules() -> List[Tuple[Any, str]]:
-    """Get processor modules for all plugins that can be used with the find_processors function
-    
-    This function adapts pluggy plugins to be compatible with the traditional find_processors system
-    
-    :return: A list of (module, processor_name) tuples
-    """
-    result = []
-    
-    # Import base modules once to avoid repeated imports
-    from changedetectionio.processors.text_json_diff import processor as text_json_diff_processor
-
-    # For each plugin, map to a suitable module for find_processors
-    for processor_name, plugin in _get_plugin_name_map().items():
-        try:
-            processor_class = _call_hook_for_plugin(plugin, 'get_processor_class')
-            
-            if processor_class:
-                # Check if this processor extends the text_json_diff processor
-                base_class_name = str(processor_class.__bases__[0].__name__)
-                if base_class_name == 'perform_site_check' or 'TextJsonDiffProcessor' in base_class_name:
-                    result.append((text_json_diff_processor, processor_name))
-        except Exception as e:
-            logger.error(f"Error mapping processor module for {processor_name}: {str(e)}")
-    
-    return result

changedetectionio/processors/text_json_diff/__init__.py

@@ -28,13 +28,13 @@ def _task(watch, update_handler):
     return text_after_filter
 
 
-def prepare_filter_prevew(datastore, watch_uuid, form_data):
+def prepare_filter_prevew(datastore, watch_uuid):
     '''Used by @app.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])'''
     from changedetectionio import forms, html_tools
     from changedetectionio.model.Watch import model as watch_model
     from concurrent.futures import ProcessPoolExecutor
     from copy import deepcopy
-    from flask import request
+    from flask import request, jsonify
     import brotli
     import importlib
     import os
@@ -50,12 +50,12 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
 
     if tmp_watch and tmp_watch.history and os.path.isdir(tmp_watch.watch_data_dir):
         # Splice in the temporary stuff from the form
-        form = forms.processor_text_json_diff_form(formdata=form_data if request.method == 'POST' else None,
-                                                   data=form_data
+        form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None,
+                                                   data=request.form
                                                    )
 
         # Only update vars that came in via the AJAX post
-        p = {k: v for k, v in form.data.items() if k in form_data.keys()}
+        p = {k: v for k, v in form.data.items() if k in request.form.keys()}
         tmp_watch.update(p)
         blank_watch_no_filters = watch_model()
         blank_watch_no_filters['url'] = tmp_watch.get('url')
@@ -103,12 +103,13 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
 
     logger.trace(f"Parsed in {time.time() - now:.3f}s")
 
-    return ({
+    return jsonify(
+        {
             'after_filter': text_after_filter,
             'before_filter': text_before_filter.decode('utf-8') if isinstance(text_before_filter, bytes) else text_before_filter,
             'duration': time.time() - now,
             'trigger_line_numbers': trigger_line_numbers,
             'ignore_line_numbers': ignore_line_numbers,
-        })
-
+        }
+    )
 

changedetectionio/processors/text_json_diff/processor.py

@@ -6,7 +6,6 @@ import os
 import re
 import urllib3
 
-from changedetectionio.conditions import execute_ruleset_against_all_plugins
 from changedetectionio.processors import difference_detection_processor
 from changedetectionio.html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text, TRANSLATE_WHITESPACE_TABLE
 from changedetectionio import html_tools, content_fetchers
@@ -332,16 +331,6 @@ class perform_site_check(difference_detection_processor):
             if result:
                 blocked = True
 
-        # And check if 'conditions' will let this pass through
-        if watch.get('conditions') and watch.get('conditions_match_logic'):
-            if not execute_ruleset_against_all_plugins(current_watch_uuid=watch.get('uuid'),
-                                                application_datastruct=self.datastore.data,
-                                                ephemeral_data={
-                                                    'text': stripped_text_from_html
-                                                }
-                                                ):
-                # Conditions say "Condition not met" so we block it.
-                blocked = True
 
         # Looks like something changed, but did it match all the rules?
         if blocked:

changedetectionio/processors/whois_plugin.py

@@ -1,169 +0,0 @@
-from loguru import logger
-import re
-import urllib.parse
-from .pluggy_interface import hookimpl
-from requests.structures import CaseInsensitiveDict
-from changedetectionio.content_fetchers.base import Fetcher
-
-# Import the text_json_diff processor
-from changedetectionio.processors.text_json_diff.processor import perform_site_check as TextJsonDiffProcessor
-
-# WHOIS Processor implementation that extends TextJsonDiffProcessor
-class WhoisProcessor(TextJsonDiffProcessor):
-    
-    def _extract_domain_from_url(self, url):
-        """Extract domain from URL, removing www. prefix if present"""
-        parsed_url = urllib.parse.urlparse(url)
-        domain = parsed_url.netloc
-        
-        # Remove www. prefix if present
-        domain = re.sub(r'^www\.', '', domain)
-        
-        return domain
-    
-    def call_browser(self, preferred_proxy_id=None):
-        """Override call_browser to perform WHOIS lookup instead of using a browser
-        
-        Note: The python-whois library doesn't directly support proxies. For real proxy support,
-        we would need to implement a custom socket connection that routes through the proxy.
-        This is a TODO for a future enhancement.
-        """
-        # Initialize a basic fetcher - this is used by the parent class
-        self.fetcher = Fetcher()
-        
-        # Extract URL from watch
-        url = self.watch.link
-        
-        # Check for file:// access
-        if re.search(r'^file:', url.strip(), re.IGNORECASE):
-            if not self.datastore.data.get('settings', {}).get('application', {}).get('allow_file_uri', False):
-                raise Exception("file:// type access is denied for security reasons.")
-        
-        # Extract domain from URL
-        domain = self._extract_domain_from_url(url)
-        
-        # Ensure we have a valid domain
-        if not domain:
-            error_msg = f"Could not extract domain from URL: '{url}'"
-            self.fetcher.content = error_msg
-            self.fetcher.status_code = 400
-            logger.error(error_msg)
-            return
-        
-        # Get proxy configuration using the common method from parent class
-        proxy_config, proxy_url = super()._get_proxy_for_watch(preferred_proxy_id)
-        
-        try:
-            # Use python-whois to get domain information
-            import whois
-            
-            # If we have proxy config, use it for the WHOIS lookup
-            # Note: The python-whois library doesn't directly support proxies,
-            # but we can implement proxy support if necessary using custom socket code
-            if proxy_config:
-                # For now, just log that we would use a proxy
-                logger.info(f"Using proxy for WHOIS lookup: {proxy_config}")
-            
-            # Perform the WHOIS lookup
-            whois_info = whois.whois(domain)
-            
-            # Convert whois_info object to text
-            if hasattr(whois_info, 'text'):
-                # Some whois implementations store raw text in .text attribute
-                whois_text = whois_info.text
-            else:
-                # Otherwise, format it nicely as key-value pairs
-                whois_text = f"WHOIS Information for domain: {domain}\n\n"
-                for key, value in whois_info.items():
-                    if value:
-                        whois_text += f"{key}: {value}\n"
-            
-            # Set the content and status for the fetcher
-            self.fetcher.content = whois_text
-            self.fetcher.status_code = 200
-            
-            # Setup headers dictionary for the fetcher
-            self.fetcher.headers = CaseInsensitiveDict({
-                'content-type': 'text/plain',
-                'server': 'whois-processor'
-            })
-            
-            # Add getters for headers
-            self.fetcher.get_all_headers = lambda: self.fetcher.headers
-            self.fetcher.get_last_status_code = lambda: self.fetcher.status_code
-            
-            # Implement necessary methods
-            self.fetcher.quit = lambda: None
-            
-        except Exception as e:
-            error_msg = f"Error fetching WHOIS data for domain {domain}: {str(e)}"
-            self.fetcher.content = error_msg
-            self.fetcher.status_code = 500
-            self.fetcher.headers = CaseInsensitiveDict({
-                'content-type': 'text/plain',
-                'server': 'whois-processor'
-            })
-            self.fetcher.get_all_headers = lambda: self.fetcher.headers
-            self.fetcher.get_last_status_code = lambda: self.fetcher.status_code
-            self.fetcher.quit = lambda: None
-            logger.error(error_msg)
-
-        return
-    
-    def run_changedetection(self, watch):
-        """Use the parent's run_changedetection which will use our overridden call_browser method"""
-        try:
-            # Let the parent class handle everything now that we've overridden call_browser
-            changed_detected, update_obj, filtered_text = super().run_changedetection(watch)
-            return changed_detected, update_obj, filtered_text
-            
-        except Exception as e:
-            error_msg = f"Error in WHOIS processor: {str(e)}"
-            update_obj = {'last_notification_error': False, 'last_error': error_msg}
-            logger.error(error_msg)
-            return False, update_obj, error_msg.encode('utf-8')
-
-    @staticmethod
-    def perform_site_check(datastore, watch_uuid):
-        """Factory method to create a WhoisProcessor instance - for compatibility with legacy code"""
-        processor = WhoisProcessor(datastore=datastore, watch_uuid=watch_uuid)
-        return processor
-
-@hookimpl
-def perform_site_check(datastore, watch_uuid):
-    """Create and return a processor instance ready to perform site check"""
-    return WhoisProcessor(datastore=datastore, watch_uuid=watch_uuid)
-
-@hookimpl(trylast=True)  # Use trylast to ensure this runs last in case of conflicts
-def get_processor_name():
-    """Return the name of this processor"""
-    from loguru import logger
-    logger.debug("whois_plugin.get_processor_name() called")
-    return "whois"
-
-@hookimpl
-def get_processor_description():
-    """Return the description of this processor"""
-    return "WHOIS Domain Information Changes Detector"
-
-@hookimpl
-def get_processor_class():
-    """Return the processor class"""
-    return WhoisProcessor
-
-@hookimpl
-def get_processor_form():
-    """Return the processor form class"""
-    # Import here to avoid circular imports
-    try:
-        from changedetectionio.forms import processor_text_json_diff_form
-        return processor_text_json_diff_form
-    except Exception as e:
-        from loguru import logger
-        logger.error(f"Error importing form for whois plugin: {str(e)}")
-        return None
-
-@hookimpl
-def get_processor_watch_model():
-    """Return the watch model class for this processor"""
-    return None  # Use default watch model

changedetectionio/static/images/copy.svg

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <svg
    version="1.1"
-   id="copy"
+   id="Layer_1"
    x="0px"
    y="0px"
    viewBox="0 0 115.77 122.88"

changedetectionio/static/images/email.svg

@@ -6,7 +6,7 @@
    height="7.5005589"
    width="11.248507"
    version="1.1"
-   id="email"
+   id="Layer_1"
    viewBox="0 0 7.1975545 4.7993639"
    xml:space="preserve"
    xmlns="http://www.w3.org/2000/svg"

changedetectionio/static/images/schedule.svg

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <svg
    version="1.1"
-   id="schedule"
+   id="Layer_1"
    x="0px"
    y="0px"
    viewBox="0 0 661.20001 665.40002"

changedetectionio/static/js/browser-steps.js

@@ -221,7 +221,7 @@ $(document).ready(function () {
                     // If you switch to "Click X,y" after an element here is setup, it will give the last co-ords anyway
                     //if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
                         $('select', first_available).val('Click element').change();
-                        $('input[type=text]', first_available).first().val(x['xpath']).focus();
+                        $('input[type=text]', first_available).first().val(x['xpath']);
                         found_something = true;
                     //}
                 }
@@ -305,7 +305,7 @@ $(document).ready(function () {
 
         if ($(this).val() === 'Click X,Y' && last_click_xy['x'] > 0 && $(elem_value).val().length === 0) {
             // @todo handle scale
-            $(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']).focus();
+            $(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']);
         }
     }).change();
 

changedetectionio/static/js/conditions.js

@@ -1,150 +0,0 @@
-$(document).ready(function () {
-    // Function to set up button event handlers
-    function setupButtonHandlers() {
-        // Unbind existing handlers first to prevent duplicates
-        $(".addRuleRow, .removeRuleRow, .verifyRuleRow").off("click");
-        
-        // Add row button handler
-        $(".addRuleRow").on("click", function(e) {
-            e.preventDefault();
-            
-            let currentRow = $(this).closest("tr");
-            
-            // Clone without events
-            let newRow = currentRow.clone(false);
-            
-            // Reset input values in the cloned row
-            newRow.find("input").val("");
-            newRow.find("select").prop("selectedIndex", 0);
-            
-            // Insert the new row after the current one
-            currentRow.after(newRow);
-            
-            // Reindex all rows
-            reindexRules();
-        });
-        
-        // Remove row button handler
-        $(".removeRuleRow").on("click", function(e) {
-            e.preventDefault();
-            
-            // Only remove if there's more than one row
-            if ($("#rulesTable tbody tr").length > 1) {
-                $(this).closest("tr").remove();
-                reindexRules();
-            }
-        });
-        
-        // Verify rule button handler
-        $(".verifyRuleRow").on("click", function(e) {
-            e.preventDefault();
-            
-            let row = $(this).closest("tr");
-            let field = row.find("select[name$='field']").val();
-            let operator = row.find("select[name$='operator']").val();
-            let value = row.find("input[name$='value']").val();
-            
-            // Validate that all fields are filled
-            if (!field || field === "None" || !operator || operator === "None" || !value) {
-                alert("Please fill in all fields (Field, Operator, and Value) before verifying.");
-                return;
-            }
-
-            
-            // Create a rule object
-            const rule = {
-                field: field,
-                operator: operator,
-                value: value
-            };
-            
-            // Show a spinner or some indication that verification is in progress
-            const $button = $(this);
-            const originalHTML = $button.html();
-            $button.html("⌛").prop("disabled", true);
-            
-            // Collect form data - similar to request_textpreview_update() in watch-settings.js
-            let formData = new FormData();
-            $('#edit-text-filter textarea, #edit-text-filter input').each(function() {
-                const $element = $(this);
-                const name = $element.attr('name');
-                if (name) {
-                    if ($element.is(':checkbox')) {
-                        formData.append(name, $element.is(':checked') ? $element.val() : false);
-                    } else {
-                        formData.append(name, $element.val());
-                    }
-                }
-            });
-            
-            // Also collect select values
-            $('#edit-text-filter select').each(function() {
-                const $element = $(this);
-                const name = $element.attr('name');
-                if (name) {
-                    formData.append(name, $element.val());
-                }
-            });
-
-
-            // Send the request to verify the rule
-            $.ajax({
-                url: verify_condition_rule_url+"?"+ new URLSearchParams({ rule: JSON.stringify(rule) }).toString(),
-                type: "POST",
-                data: formData,
-                processData: false, // Prevent jQuery from converting FormData to a string
-                contentType: false, // Let the browser set the correct content type
-                success: function (response) {
-                    if (response.status === "success") {
-                        if (response.result) {
-                            alert("✅ Condition PASSES verification against current snapshot!");
-                        } else {
-                            alert("❌ Condition FAILS verification against current snapshot.");
-                        }
-                    } else {
-                        alert("Error: " + response.message);
-                    }
-                    $button.html(originalHTML).prop("disabled", false);
-                },
-                error: function (xhr) {
-                    let errorMsg = "Error verifying condition.";
-                    if (xhr.responseJSON && xhr.responseJSON.message) {
-                        errorMsg = xhr.responseJSON.message;
-                    }
-                    alert(errorMsg);
-                    $button.html(originalHTML).prop("disabled", false);
-                }
-            });
-        });
-    }
-
-    // Function to reindex form elements and re-setup event handlers
-    function reindexRules() {
-        // Unbind all button handlers first
-        $(".addRuleRow, .removeRuleRow, .verifyRuleRow").off("click");
-        
-        // Reindex all form elements
-        $("#rulesTable tbody tr").each(function(index) {
-            $(this).find("select, input").each(function() {
-                let oldName = $(this).attr("name");
-                let oldId = $(this).attr("id");
-
-                if (oldName) {
-                    let newName = oldName.replace(/\d+/, index);
-                    $(this).attr("name", newName);
-                }
-
-                if (oldId) {
-                    let newId = oldId.replace(/\d+/, index);
-                    $(this).attr("id", newId);
-                }
-            });
-        });
-        
-        // Reattach event handlers after reindexing
-        setupButtonHandlers();
-    }
-
-    // Initial setup of button handlers
-    setupButtonHandlers();
-});

changedetectionio/static/js/notifications.js

@@ -1,52 +1,42 @@
-$(document).ready(function () {
+$(document).ready(function() {
 
-    $('#add-email-helper').click(function (e) {
-        e.preventDefault();
-        email = prompt("Destination email");
-        if (email) {
-            var n = $(".notification-urls");
-            var p = email_notification_prefix;
-            $(n).val($.trim($(n).val()) + "\n" + email_notification_prefix + email);
+  $('#add-email-helper').click(function (e) {
+    e.preventDefault();
+    email = prompt("Destination email");
+    if(email) {
+      var n = $(".notification-urls");
+      var p=email_notification_prefix;
+      $(n).val( $.trim( $(n).val() )+"\n"+email_notification_prefix+email );
+    }
+  });
+
+  $('#send-test-notification').click(function (e) {
+    e.preventDefault();
+
+    data = {
+      notification_body: $('#notification_body').val(),
+      notification_format: $('#notification_format').val(),
+      notification_title: $('#notification_title').val(),
+      notification_urls: $('.notification-urls').val(),
+      tags: $('#tags').val(),
+      window_url: window.location.href,
+    }
+
+
+    $.ajax({
+      type: "POST",
+      url: notification_base_url,
+      data : data,
+        statusCode: {
+        400: function(data) {
+          // More than likely the CSRF token was lost when the server restarted
+          alert(data.responseText);
         }
-    });
-
-    $('#send-test-notification').click(function (e) {
-        e.preventDefault();
-
-        data = {
-            notification_body: $('#notification_body').val(),
-            notification_format: $('#notification_format').val(),
-            notification_title: $('#notification_title').val(),
-            notification_urls: $('.notification-urls').val(),
-            tags: $('#tags').val(),
-            window_url: window.location.href,
-        }
-
-        $('.notifications-wrapper .spinner').fadeIn();
-        $('#notification-test-log').show();
-        $.ajax({
-            type: "POST",
-            url: notification_base_url,
-            data: data,
-            statusCode: {
-                400: function (data) {
-                    $("#notification-test-log>span").text(data.responseText);
-                },
-            }
-        }).done(function (data) {
-            $("#notification-test-log>span").text(data);
-        }).fail(function (jqXHR, textStatus, errorThrown) {
-            // Handle connection refused or other errors
-            if (textStatus === "error" && errorThrown === "") {
-                console.error("Connection refused or server unreachable");
-                $("#notification-test-log>span").text("Error: Connection refused or server is unreachable.");
-            } else {
-                console.error("Error:", textStatus, errorThrown);
-                $("#notification-test-log>span").text("An error occurred: " + textStatus);
-            }
-        }).always(function () {
-            $('.notifications-wrapper .spinner').hide();
-        })
-    });
+      }
+    }).done(function(data){
+      console.log(data);
+      alert(data);
+    })
+  });
 });
 

changedetectionio/static/js/tabs.js

@@ -26,6 +26,7 @@ function set_active_tab() {
     if (tab.length) {
         tab[0].parentElement.className = "active";
     }
+
 }
 
 function focus_error_tab() {

changedetectionio/static/styles/scss/parts/_browser-steps.scss

@@ -40,22 +40,19 @@
   }
 }
 
-@media only screen and (min-width: 760px) {
 
-  #browser-steps .flex-wrapper {
-    display: flex;
-    flex-flow: row;
-    height: 70vh;
-    font-size: 80%;
+#browser-steps .flex-wrapper {
+  display: flex;
+  flex-flow: row;
+  height: 70vh;
+  font-size: 80%;
+  #browser-steps-ui {
+    flex-grow: 1;      /* Allow it to grow and fill the available space */
+    flex-shrink: 1;    /* Allow it to shrink if needed */
+    flex-basis: 0;     /* Start with 0 base width so it stretches as much as possible */
+    background-color: #eee;
+    border-radius: 5px;
 
-    #browser-steps-ui {
-      flex-grow: 1; /* Allow it to grow and fill the available space */
-      flex-shrink: 1; /* Allow it to shrink if needed */
-      flex-basis: 0; /* Start with 0 base width so it stretches as much as possible */
-      background-color: #eee;
-      border-radius: 5px;
-
-    }
   }
 
   #browser-steps-fieldlist {
@@ -66,21 +63,15 @@
     padding-left: 1rem;
     overflow-y: scroll;
   }
-
-  /*  this is duplicate :( */
-  #browsersteps-selector-wrapper {
-    height: 100% !important;
-  }
 }
 
 /*  this is duplicate :( */
 #browsersteps-selector-wrapper {
-
+  height: 100%;
   width: 100%;
   overflow-y: scroll;
   position: relative;
-  height: 80vh;
-
+  //width: 100%;
   > img {
     position: absolute;
     max-width: 100%;
@@ -100,6 +91,7 @@
     left: 50%;
     top: 50%;
     transform: translate(-50%, -50%);
+    margin-left: -40px;
     z-index: 100;
     max-width: 350px;
     text-align: center;

changedetectionio/static/styles/scss/parts/_edit.scss

@@ -1,9 +0,0 @@
-ul#conditions_match_logic {
-    list-style: none;
-  input, label, li {
-    display: inline-block;
-  }
-  li {
-    padding-right: 1em;
-  }
-}

changedetectionio/static/styles/scss/styles.scss

@@ -13,7 +13,6 @@
 @import "parts/_menu";
 @import "parts/_love";
 @import "parts/preview_text_filter";
-@import "parts/_edit";
 
 body {
   color: var(--color-text);
@@ -381,15 +380,7 @@ a.pure-button-selected {
 }
 
 .notifications-wrapper {
-  padding-top: 0.5rem;
-  #notification-test-log {
-    padding-top: 1rem;
-    white-space: pre-wrap;
-    word-break: break-word;
-    overflow-wrap: break-word;
-    max-width: 100%;
-    box-sizing: border-box;
-  }
+  padding: 0.5rem 0 1rem 0;
 }
 
 label {

changedetectionio/static/styles/styles.css

@@ -46,22 +46,21 @@
     #browser_steps li > label {
       display: none; }
 
-@media only screen and (min-width: 760px) {
-  #browser-steps .flex-wrapper {
-    display: flex;
-    flex-flow: row;
-    height: 70vh;
-    font-size: 80%; }
-    #browser-steps .flex-wrapper #browser-steps-ui {
-      flex-grow: 1;
-      /* Allow it to grow and fill the available space */
-      flex-shrink: 1;
-      /* Allow it to shrink if needed */
-      flex-basis: 0;
-      /* Start with 0 base width so it stretches as much as possible */
-      background-color: #eee;
-      border-radius: 5px; }
-  #browser-steps-fieldlist {
+#browser-steps .flex-wrapper {
+  display: flex;
+  flex-flow: row;
+  height: 70vh;
+  font-size: 80%; }
+  #browser-steps .flex-wrapper #browser-steps-ui {
+    flex-grow: 1;
+    /* Allow it to grow and fill the available space */
+    flex-shrink: 1;
+    /* Allow it to shrink if needed */
+    flex-basis: 0;
+    /* Start with 0 base width so it stretches as much as possible */
+    background-color: #eee;
+    border-radius: 5px; }
+  #browser-steps .flex-wrapper #browser-steps-fieldlist {
     flex-grow: 0;
     /* Don't allow it to grow */
     flex-shrink: 0;
@@ -72,16 +71,13 @@
     /* Set a max width to prevent overflow */
     padding-left: 1rem;
     overflow-y: scroll; }
-  /*  this is duplicate :( */
-  #browsersteps-selector-wrapper {
-    height: 100% !important; } }
 
 /*  this is duplicate :( */
 #browsersteps-selector-wrapper {
+  height: 100%;
   width: 100%;
   overflow-y: scroll;
   position: relative;
-  height: 80vh;
   /* nice tall skinny one */ }
   #browsersteps-selector-wrapper > img {
     position: absolute;
@@ -96,6 +92,7 @@
     left: 50%;
     top: 50%;
     transform: translate(-50%, -50%);
+    margin-left: -40px;
     z-index: 100;
     max-width: 350px;
     text-align: center; }
@@ -523,13 +520,6 @@ body.preview-text-enabled {
   z-index: 3;
   box-shadow: 1px 1px 4px var(--color-shadow-jump); }
 
-ul#conditions_match_logic {
-  list-style: none; }
-  ul#conditions_match_logic input, ul#conditions_match_logic label, ul#conditions_match_logic li {
-    display: inline-block; }
-  ul#conditions_match_logic li {
-    padding-right: 1em; }
-
 body {
   color: var(--color-text);
   background: var(--color-background-page);
@@ -790,14 +780,7 @@ a.pure-button-selected {
   cursor: pointer; }
 
 .notifications-wrapper {
-  padding-top: 0.5rem; }
-  .notifications-wrapper #notification-test-log {
-    padding-top: 1rem;
-    white-space: pre-wrap;
-    word-break: break-word;
-    overflow-wrap: break-word;
-    max-width: 100%;
-    box-sizing: border-box; }
+  padding: 0.5rem 0 1rem 0; }
 
 label:hover {
   cursor: pointer; }

changedetectionio/templates/_common_fields.html

@@ -12,25 +12,23 @@
                             }}
                             <div class="pure-form-message-inline">
                                 <p>
-                                <strong>Tip:</strong> Use <a target="newwindow" href="https://github.com/caronc/apprise">AppRise Notification URLs</a> for notification to just about any service! <i><a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.<br>
+                                <strong>Tip:</strong> Use <a target=_new href="https://github.com/caronc/apprise">AppRise Notification URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.<br>
 </p>
                                 <div data-target="#advanced-help-notifications" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</div>
                                 <ul style="display: none" id="advanced-help-notifications">
-                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
-                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
-                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
                                 <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>) <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes#postposts">more help here</a></li>
                                   <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                               </ul>
                             </div>
                             <div class="notifications-wrapper">
-                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a> <div class="spinner"  style="display: none;"></div>
+                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a>
                             {% if emailprefix %}
                               <a id="add-email-helper" class="pure-button button-secondary button-xsmall" >Add email <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='email.svg')}}" alt="Add an email address"> </a>
                             {% endif %}
                               <a href="{{url_for('notification_logs')}}" class="pure-button button-secondary button-xsmall" >Notification debug logs</a>
-                              <br>
-                                <div id="notification-test-log" style="display: none;"><span class="pure-form-message-inline">Processing..</span></div>
                             </div>
                         </div>
                         <div id="notification-customisation" class="pure-control-group">
@@ -40,7 +38,7 @@
                             </div>
                             <div class="pure-control-group">
                                 {{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }}
-                                <span class="pure-form-message-inline">Body for all notifications &dash; You can use <a target="newwindow" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
+                                <span class="pure-form-message-inline">Body for all notifications &dash; You can use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
                                 </span>
 
                             </div>
@@ -126,7 +124,7 @@
                                 <div class="pure-form-message-inline">
                                     <p>
 									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. <br>
-                                    For example, an addition or removal could be perceived as a change in some cases. <a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
+                                    For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
                                     </p>
                                     <p>
                                         For JSON payloads, use <strong>|tojson</strong> without quotes for automatic escaping, for example - <code>{ "name": {{ '{{ watch_title|tojson }}' }} }</code>

changedetectionio/templates/_helpers.html

@@ -61,55 +61,6 @@
   {{ field(**kwargs)|safe }}
 {% endmacro %}
 
-{% macro render_fieldlist_of_formfields_as_table(fieldlist, table_id="rulesTable") %}
-  <table class="fieldlist_formfields pure-table" id="{{ table_id }}">
-    <thead>
-      <tr>
-        {% for subfield in fieldlist[0] %}
-          <th>{{ subfield.label }}</th>
-        {% endfor %}
-        <th>Actions</th>
-      </tr>
-    </thead>
-    <tbody>
-      {% for form_row in fieldlist %}
-        <tr {% if form_row.errors %} class="error-row" {% endif %}>
-          {% for subfield in form_row %}
-            <td>
-              {{ subfield()|safe }}
-              {% if subfield.errors %}
-                <ul class="errors">
-                  {% for error in subfield.errors %}
-                    <li class="error">{{ error }}</li>
-                  {% endfor %}
-                </ul>
-              {% endif %}
-            </td>
-          {% endfor %}
-          <td>
-            <button type="button" class="addRuleRow">+</button>
-            <button type="button" class="removeRuleRow">-</button>
-            <button type="button" class="verifyRuleRow" title="Verify this rule against current snapshot">✓</button>
-          </td>
-        </tr>
-      {% endfor %}
-    </tbody>
-  </table>
-{% endmacro %}
-
-
-{% macro playwright_warning() %}
-    <p><strong>Error - Playwright support for Chrome based fetching is not enabled.</strong> Alternatively try our <a href="https://changedetection.io">very affordable subscription based service which has all this setup for you</a>.</p>
-    <p>You may need to <a href="https://github.com/dgtlmoon/changedetection.io/blob/09ebc6ec6338545bdd694dc6eee57f2e9d2b8075/docker-compose.yml#L31">Enable playwright environment variable</a> and uncomment the <strong>sockpuppetbrowser</strong> in the <a href="https://github.com/dgtlmoon/changedetection.io/blob/master/docker-compose.yml">docker-compose.yml</a> file.</p>
-    <br>
-    <p>(Also Selenium/WebDriver can not extract full page screenshots reliably so Playwright is recommended here)</p>
-
-{% endmacro %}
-
-{% macro only_webdriver_type_watches_warning() %}
-    <p><strong>Sorry, this functionality only works with Playwright/Chrome enabled watches.<br>You need to <a href="#request">Set the fetch method to Playwright/Chrome mode and resave</a> and have the Playwright connection enabled.</strong></p><br>
-{% endmacro %}
-
 {% macro render_time_schedule_form(form, available_timezones, timezone_default_config) %}
     <style>
     .day-schedule *, .day-schedule select {

changedetectionio/templates/base.html

@@ -159,7 +159,7 @@
                     <a id="chrome-extension-link"
                        title="Try our new Chrome Extension!"
                        href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                        <img alt="Chrome store icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}">
+                        <img src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}">
                         Chrome Webstore
                     </a>
                 </p>

changedetectionio/templates/edit.html

@@ -1,14 +1,11 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, playwright_warning, only_webdriver_type_watches_warning, render_fieldlist_of_formfields_as_table %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='scheduler.js')}}" defer></script>
-<script src="{{url_for('static_content', group='js', filename='conditions.js')}}" defer></script>
-
-
 <script>
     const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
@@ -43,17 +40,17 @@
 
     <div class="tabs collapsable">
         <ul>
-            <li class="tab"><a href="#general">General</a></li>
+            <li class="tab" id=""><a href="#general">General</a></li>
             <li class="tab"><a href="#request">Request</a></li>
             {% if extra_tab_content %}
             <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
             {% endif %}
+            {% if playwright_enabled %}
             <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
-        <!-- should goto extra forms? -->
+            {% endif %}
             {% if watch['processor'] == 'text_json_diff' %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
             <li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
-            <li class="tab" id="conditions-tab"><a href="#conditions">Conditions</a></li>
             {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
             <li class="tab"><a href="#stats">Stats</a></li>
@@ -202,9 +199,8 @@ Math: {{ 1 + 1 }}") }}
                     </div>
             </fieldset>
             </div>
-
+            {% if playwright_enabled %}
             <div class="tab-pane-inner" id="browser-steps">
-            {% if playwright_enabled and watch_uses_webdriver %}
                 <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
                 <fieldset>
                     <div class="pure-control-group">
@@ -228,7 +224,7 @@ Math: {{ 1 + 1 }}") }}
                                     <span class="loader" >
                                         <span id="browsersteps-click-start">
                                             <h2 >Click here to Start</h2>
-                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="start"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
+                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="Layer_1"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
                                             Please allow 10-15 seconds for the browser to connect.<br>
                                         </span>
                                         <div class="spinner"  style="display: none;"></div>
@@ -238,31 +234,21 @@ Math: {{ 1 + 1 }}") }}
                                 </div>
                             </div>
                             <div id="browser-steps-fieldlist" >
-                                <span id="browser-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
+                                <span id="browser-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target=_new href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
                                 {{ render_field(form.browser_steps) }}
                             </div>
                         </div>
                     </div>
                 </fieldset>
-                {% else %}
-                    <span class="pure-form-message-inline">
-                        {% if not watch_uses_webdriver %}
-                            {{ only_webdriver_type_watches_warning() }}
-                        {% endif %}
-                        {%  if not playwright_enabled %}
-                            {{ playwright_warning() }}
-                        {% endif %}
-                    </span>
-                {% endif %}
             </div>
-
+            {% endif %}
 
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div  class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_muted) }}
                     </div>
-                    {% if watch_uses_webdriver %}
+                    {% if is_html_webdriver %}
                     <div class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_screenshot) }}
                         <span class="pure-form-message-inline">
@@ -278,39 +264,13 @@ Math: {{ 1 + 1 }}") }}
                         </div>
                         {% endif %}
                         <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
+
                         {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
                     </div>
                 </fieldset>
             </div>
 
             {% if watch['processor'] == 'text_json_diff' %}
-
-            <div class="tab-pane-inner" id="conditions">
-                    <script>
-                        const verify_condition_rule_url="{{url_for('conditions.verify_condition_single_rule', watch_uuid=uuid)}}";
-                    </script>
-                <style>
-                    .verifyRuleRow {
-                        background-color: #4caf50;
-                        color: white;
-                        border: none;
-                        cursor: pointer;
-                        font-weight: bold;
-                    }
-                    .verifyRuleRow:hover {
-                        background-color: #45a049;
-                    }
-                </style>
-                <div class="pure-control-group">
-                    {{ render_field(form.conditions_match_logic) }}
-                    {{ render_fieldlist_of_formfields_as_table(form.conditions) }}
-                    <div class="pure-form-message-inline">
-                        <br>
-                        Use the verify (✓) button to test if a condition passes against the current snapshot.<br><br>
-                        Did you know that <strong>conditions</strong> can be extended with your own custom plugin? tutorials coming soon!<br>
-                    </div>
-                </div>
-            </div>
             <div class="tab-pane-inner" id="filters-and-triggers">
                 <span id="activate-text-preview" class="pure-button pure-button-primary button-xsmall">Activate preview</span>
               <div>
@@ -338,7 +298,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                           <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
                         {% endif %}
                         <span class="pure-form-message-inline">One CSS, xPath, JSON Path/JQ selector per line, <i>any</i> rules that matches will be used.<br>
-                        <span data-target="#advanced-help-selectors" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</span><br>
+<p><div data-target="#advanced-help-selectors" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</div><br></p>
                     <ul id="advanced-help-selectors" style="display: none;">
                         <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
                         <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
@@ -511,7 +471,7 @@ keyword") }}
 
                 <fieldset>
                     <div class="pure-control-group">
-                        {% if playwright_enabled and watch_uses_webdriver %}
+                        {% if visualselector_enabled %}
                             <span class="pure-form-message-inline" id="visual-selector-heading">
                                 The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection. It automatically fills-in the filters in the "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab. Use <strong>Shift+Click</strong> to select multiple items.
                             </span>
@@ -529,12 +489,11 @@ keyword") }}
                             </div>
                             <div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
                         {% else %}
-                            {% if not watch_uses_webdriver %}
-                                {{ only_webdriver_type_watches_warning() }}
-                            {% endif %}
-                            {% if not playwright_enabled %}
-                                {{ playwright_warning() }}
-                            {% endif %}
+                            <span class="pure-form-message-inline">
+                                <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
+                                <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
+                                <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
+                            </span>
                         {% endif %}
                     </div>
                 </fieldset>

changedetectionio/templates/settings.html

@@ -214,7 +214,7 @@ nav
                         <a id="chrome-extension-link"
                            title="Try our new Chrome Extension!"
                            href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                            <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}" alt="Chrome">
+                            <img src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}" alt="Chrome">
                             Chrome Webstore
                         </a>
                     </p>
@@ -280,7 +280,9 @@ nav
                         
                     </div>
                 </div>
-
+                <p>
+                    Your proxy provider may need to whitelist our IP of <code>204.15.192.195</code>
+                </p>
                <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
 
                 <div class="pure-control-group" id="extra-proxies-setting">

changedetectionio/templates/svgs/search-icon.svg

@@ -1 +1 @@
-<svg version="1.1" id="search" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>

changedetectionio/templates/watch-overview.html

@@ -108,8 +108,7 @@
                     {% else %}
                     <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
                     {% endif %}
-                    {% set mute_label = 'UnMute notification' if watch.notification_muted else 'Mute notification' %}
-                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="{{ mute_label }}" title="{{ mute_label }}" class="icon icon-mute" ></a>
+                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
                 </td>
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
@@ -119,7 +118,7 @@
                          or (  watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
                          or "extra_browser_" in watch.get_fetch_backend
                     %}
-                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" alt="Using a Chrome browser" title="Using a Chrome browser" >
+                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" title="Using a Chrome browser" >
                     {% endif %}
 
                     {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" >{% endif %}
@@ -192,7 +191,7 @@
                     {% if watch.history_n >= 2 %}
 
                         {%  if is_unviewed %}
-                           <a href="{{ url_for('diff_history_page', uuid=watch.uuid, from_version=watch.get_from_version_based_on_last_viewed) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
+                           <a href="{{ url_for('diff_history_page', uuid=watch.uuid, from_version=watch.get_next_snapshot_key_to_last_viewed) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
                         {% else %}
                            <a href="{{ url_for('diff_history_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
                         {% endif %}

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -34,7 +34,7 @@ def test_execute_custom_js(client, live_server, measure_memory_usage):
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
 
     assert b"This text should be removed" not in res.data

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -48,7 +48,7 @@ def test_noproxy_option(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.get(
         url_for("edit_page", uuid=uuid, unpause_on_save=1))
     assert b'No proxy' in res.data

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -81,7 +81,7 @@ def test_socks5(client, live_server, measure_memory_usage):
     assert "Awesome, you made it".encode('utf-8') in res.data
 
     # PROXY CHECKER WIDGET CHECK - this needs more checking
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     res = client.get(
         url_for("check_proxies.start_check", uuid=uuid),

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
-
 import os.path
-
+import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output
+from changedetectionio import html_tools
 
 
 def set_original(excluding=None, add_line=None):
@@ -165,7 +165,6 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
-
     assert b'unviewed' in res.data
 
     # Takes a moment for apprise to fire

changedetectionio/tests/test_api.py

@@ -44,6 +44,7 @@ def set_modified_response():
 
     return None
 
+
 def is_valid_uuid(val):
     try:
         uuid.UUID(str(val))
@@ -55,9 +56,8 @@ def is_valid_uuid(val):
 def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-
 def test_api_simple(client, live_server, measure_memory_usage):
-#    live_server_setup(live_server)
+    #live_server_setup(live_server)
 
     api_key = extract_api_key_from_UI(client)
 
@@ -129,9 +129,6 @@ def test_api_simple(client, live_server, measure_memory_usage):
     assert after_recheck_info['last_checked'] != before_recheck_info['last_checked']
     assert after_recheck_info['last_changed'] != 0
 
-    # #2877 When run in a slow fetcher like playwright etc
-    assert after_recheck_info['last_changed'] ==  after_recheck_info['last_checked']
-
     # Check history index list
     res = client.get(
         url_for("watchhistory", uuid=watch_uuid),

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -99,7 +99,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
     assert b'ldjson-price-track-offer' in res.data
 
     # Accept it
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     #time.sleep(1)
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
     client.get(url_for("form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_backend.py

@@ -2,6 +2,7 @@
 
 import time
 from flask import url_for
+from urllib.request import urlopen
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
     extract_UUID_from_client
 
@@ -68,7 +69,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     wait_for_all_checks(client)
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     # Check the 'get latest snapshot works'
     res = client.get(url_for("watch_get_latest_html", uuid=uuid))

changedetectionio/tests/test_conditions.py

@@ -1,133 +0,0 @@
-#!/usr/bin/env python3
-
-from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
-
-def set_original_response(number="50"):
-    test_return_data = f"""<html>
-       <body>
-     <h1>Test Page for Conditions</h1>
-     <p>This page contains a number that will be tested with conditions.</p>
-     <div class="number-container">Current value: {number}</div>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-def set_number_in_range_response(number="75"):
-    test_return_data = f"""<html>
-       <body>
-     <h1>Test Page for Conditions</h1>
-     <p>This page contains a number that will be tested with conditions.</p>
-     <div class="number-container">Current value: {number}</div>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-def set_number_out_of_range_response(number="150"):
-    test_return_data = f"""<html>
-       <body>
-     <h1>Test Page for Conditions</h1>
-     <p>This page contains a number that will be tested with conditions.</p>
-     <div class="number-container">Current value: {number}</div>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-
-def test_conditions_with_text_and_number(client, live_server):
-    """Test that both text and number conditions work together with AND logic."""
-    
-    set_original_response("50")
-    live_server_setup(live_server)
-
-    test_url = url_for('test_endpoint', _external=True)
-
-    # Add our URL to the import page
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-
-    # Configure the watch with two conditions connected with AND:
-    # 1. The page filtered text must contain "5" (first digit of value)
-    # 2. The extracted number should be >= 20 and <= 100
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-            "url": test_url,
-            "fetch_backend": "html_requests",
-            "include_filters": ".number-container",
-            "title": "Number AND Text Condition Test",
-            "conditions_match_logic": "ALL",  # ALL = AND logic
-            "conditions-0-operator": "in",
-            "conditions-0-field": "page_filtered_text",
-            "conditions-0-value": "5",
-
-            "conditions-1-operator": ">=",
-            "conditions-1-field": "extracted_number",
-            "conditions-1-value": "20",
-
-            "conditions-2-operator": "<=",
-            "conditions-2-field": "extracted_number",
-            "conditions-2-value": "100",
-
-            # So that 'operations' from pluggy discovery are tested
-            "conditions-3-operator": "length_min",
-            "conditions-3-field": "page_filtered_text",
-            "conditions-3-value": "1",
-
-            # So that 'operations' from pluggy discovery are tested
-            "conditions-4-operator": "length_max",
-            "conditions-4-field": "page_filtered_text",
-            "conditions-4-value": "100",
-
-            # So that 'operations' from pluggy discovery are tested
-            "conditions-5-operator": "contains_regex",
-            "conditions-5-field": "page_filtered_text",
-            "conditions-5-value": "\d",
-        },
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-
-    wait_for_all_checks(client)
-    client.get(url_for("mark_all_viewed"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    # Case 1
-    set_number_in_range_response("70.5")
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    # 75 is > 20 and < 100 and contains "5"
-    res = client.get(url_for("index"))
-    assert b'unviewed' in res.data
-
-
-    # Case 2: Change with one condition violated
-    # Number out of range (150) but contains '5'
-    client.get(url_for("mark_all_viewed"), follow_redirects=True)
-    set_number_out_of_range_response("150.5")
-
-
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    # Should NOT be marked as having changes since not all conditions are met
-    res = client.get(url_for("index"))
-    assert b'unviewed' not in res.data
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/tests/test_encoding.py

@@ -40,7 +40,7 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage):
 
 
     # Content type recording worked
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['content-type'] == "text/html"
 
     res = client.get(

changedetectionio/tests/test_filter_failure_notification.py

@@ -51,7 +51,7 @@ def run_filter_test(client, live_server, content_filter):
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 0, "No filter = No filter failure"
 

changedetectionio/tests/test_group.py

@@ -288,7 +288,7 @@ def test_clone_tag_on_import(client, live_server, measure_memory_usage):
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
-    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    watch_uuid = extract_UUID_from_client(client)
     res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
 
     assert b'Cloned' in res.data
@@ -315,7 +315,7 @@ def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usa
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
-    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    watch_uuid = extract_UUID_from_client(client)
     res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
 
     assert b'Cloned' in res.data

changedetectionio/tests/test_ignore.py

@@ -36,7 +36,7 @@ def test_ignore(client, live_server, measure_memory_usage):
 
     # Give the thread time to pick it up
     wait_for_all_checks(client)
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     # use the highlighter endpoint
     res = client.post(
         url_for("highlight_submit_ignore_url", uuid=uuid),

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -514,15 +514,3 @@ def test_check_jq_ext_filter(client, live_server, measure_memory_usage):
 def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage):
     if jq_support:
         check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
-
-def test_jsonpath_BOM_utf8(client, live_server, measure_memory_usage):
-    from .. import html_tools
-
-    # JSON string with BOM and correct double-quoted keys
-    json_str = '\ufeff{"name": "José", "emoji": "😊", "language": "中文", "greeting": "Привет"}'
-
-    # See that we can find the second <script> one, which is not broken, and matches our filter
-    text = html_tools.extract_json_as_string(json_str, "json:$.name")
-    assert text == '"José"'
-
-    

changedetectionio/tests/test_live_preview.py

@@ -29,7 +29,7 @@ def test_content_filter_live_preview(client, live_server, measure_memory_usage):
         data={"url": test_url, "tags": ''},
         follow_redirects=True
     )
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.post(
         url_for("edit_page", uuid=uuid),
         data={

changedetectionio/tests/test_nonrenderable_pages.py

@@ -48,7 +48,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     #####################
     client.post(
         url_for("settings_page"),
-        data={"application-empty_pages_are_a_change": "", # default, OFF, they are NOT a change
+        data={"application-empty_pages_are_a_change": "",
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
@@ -66,14 +66,6 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
-
-    assert watch.last_changed == 0
-    assert watch['last_checked'] != 0
-
-
-
 
     # ok now do the opposite
 
@@ -100,10 +92,6 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # A totally zero byte (#2528) response should also not trigger an error
     set_zero_byte_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
-    # 2877
-    assert watch.last_changed == watch['last_checked']
-
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data # A change should have registered because empty_pages_are_a_change is ON

changedetectionio/tests/test_notification.py

@@ -6,7 +6,7 @@ from flask import url_for
 from loguru import logger
 
 from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \
-    set_longer_modified_response, get_index
+    set_longer_modified_response
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -29,7 +29,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
     # Re 360 - new install should have defaults set
     res = client.get(url_for("settings_page"))
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')+"?status_code=204"
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
 
     assert default_notification_body.encode() in res.data
     assert default_notification_title.encode() in res.data
@@ -76,7 +76,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
     testimage_png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='
 
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     datastore = 'test-datastore'
     with open(os.path.join(datastore, str(uuid), 'last-screenshot.png'), 'wb') as f:
         f.write(base64.b64decode(testimage_png))
@@ -135,14 +135,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
     time.sleep(3)
-
-    # Check no errors were recorded
-    res = client.get(url_for("index"))
-    assert b'notification-error' not in res.data
-
-
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
         notification_submission = f.read()
@@ -291,7 +284,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
 
     res = client.post(
         url_for("settings_page"),
@@ -326,11 +319,6 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
 
     time.sleep(2) # plus extra delay for notifications to fire
 
-
-    # Check no errors were recorded, because we asked for 204 which is slightly uncommon but is still OK
-    res = get_index(client)
-    assert b'notification-error' not in res.data
-
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
         j = json.loads(x)
@@ -372,10 +360,7 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     #live_server_setup(live_server)
     set_original_response()
     if os.path.isfile("test-datastore/notification.txt"):
-        os.unlink("test-datastore/notification.txt") \
-
-    # 1995 UTF-8 content should be encoded
-    test_body = 'change detection is cool 网站监测 内容更新了'
+        os.unlink("test-datastore/notification.txt")
 
     # otherwise other settings would have already existed from previous tests in this file
     res = client.post(
@@ -383,7 +368,8 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
         data={
             "application-fetch_backend": "html_requests",
             "application-minutes_between_check": 180,
-            "application-notification_body": test_body,
+            #1995 UTF-8 content should be encoded
+            "application-notification_body": 'change detection is cool 网站监测 内容更新了',
             "application-notification_format": default_notification_format,
             "application-notification_urls": "",
             "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
@@ -413,10 +399,12 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert res.status_code != 400
     assert res.status_code != 500
 
+    # Give apprise time to fire
+    time.sleep(4)
 
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
-        assert test_body in x
+        assert 'change detection is cool 网站监测 内容更新了' in x
 
     os.unlink("test-datastore/notification.txt")
 

changedetectionio/tests/test_processor_registry.py

@@ -1,59 +0,0 @@
-import pytest
-from changedetectionio.processors.processor_registry import get_processor_class, get_all_processors
-
-
-def test_get_all_processors():
-    """Test that get_all_processors returns a list of processor tuples"""
-    processors = get_all_processors()
-    assert isinstance(processors, list)
-    assert len(processors) > 0
-    
-    # Each item should be a tuple of (name, description)
-    for processor in processors:
-        assert isinstance(processor, tuple)
-        assert len(processor) == 2
-        assert isinstance(processor[0], str)
-        assert isinstance(processor[1], str)
-        
-    # Check that our WHOIS processor is included
-    whois_processor = next((p for p in processors if p[0] == "whois"), None)
-    assert whois_processor is not None
-    assert whois_processor[1] == "WHOIS Domain Information Changes"
-
-
-def test_get_processor_class():
-    """Test that get_processor_class returns the right class"""
-    # Get the WHOIS processor class
-    processor_class = get_processor_class("whois")
-    assert processor_class is not None
-    
-    # It should have perform_site_check method
-    assert hasattr(processor_class, 'perform_site_check')
-    
-    # Check for non-existent processor
-    non_existent = get_processor_class("non_existent_processor")
-    assert non_existent is None
-
-
-def test_get_processor_site_check():
-    """Test that get_processor_site_check returns a processor instance"""
-    from unittest.mock import MagicMock
-    from changedetectionio.processors.processor_registry import get_processor_site_check
-    
-    # Get a WHOIS processor instance
-    mock_datastore = MagicMock()
-    watch_uuid = "test-uuid"
-    processor = get_processor_site_check("whois", mock_datastore, watch_uuid)
-    
-    # It should be a processor instance
-    assert processor is not None
-    
-    # It should have the run_changedetection method
-    assert hasattr(processor, 'run_changedetection')
-    
-    # It should have the call_browser method
-    assert hasattr(processor, 'call_browser')
-    
-    # Check for non-existent processor
-    non_existent = get_processor_site_check("non_existent_processor", mock_datastore, watch_uuid)
-    assert non_existent is None

changedetectionio/tests/test_request.py

@@ -373,14 +373,13 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     with open('test-datastore/headers-testtag.txt', 'w') as f:
-        f.write("tag-header: test\r\nurl-header: http://example.com")
+        f.write("tag-header: test")
 
     with open('test-datastore/headers.txt', 'w') as f:
-        f.write("global-header: nice\r\nnext-global-header: nice\r\nurl-header-global: http://example.com/global")
+        f.write("global-header: nice\r\nnext-global-header: nice")
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-    with open(f'test-datastore/{uuid}/headers.txt', 'w') as f:
-        f.write("watch-header: nice\r\nurl-header-watch: http://example.com/watch")
+    with open('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt', 'w') as f:
+        f.write("watch-header: nice")
 
     wait_for_all_checks(client)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -411,9 +410,6 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     assert b"Xxx:ooo" in res.data
     assert b"Watch-Header:nice" in res.data
     assert b"Tag-Header:test" in res.data
-    assert b"Url-Header:http://example.com" in res.data
-    assert b"Url-Header-Global:http://example.com/global" in res.data
-    assert b"Url-Header-Watch:http://example.com/watch" in res.data
 
     # Check the custom UA from system settings page made it through
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):

changedetectionio/tests/test_restock_itemprop.py

@@ -189,17 +189,6 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
 
     client.get(url_for("mark_all_viewed"))
 
-
-    # 2715 - Price detection (once it crosses the "lower" threshold) again with a lower price - should trigger again!
-    set_original_response(props_markup=instock_props[0], price='820.45')
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-    assert b'820.45' in res.data
-    assert b'unviewed' in res.data
-    client.get(url_for("mark_all_viewed"))
-
     # price changed to something MORE than max (1100.10), SHOULD be a change
     set_original_response(props_markup=instock_props[0], price='1890.45')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -214,7 +203,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
 
 
 def test_restock_itemprop_minmax(client, live_server):
-    #live_server_setup(live_server)
+#    live_server_setup(live_server)
     extras = {
         "restock_settings-follow_price_changes": "y",
         "restock_settings-price_change_min": 900.0,
@@ -380,7 +369,7 @@ def test_change_with_notification_values(client, live_server):
 
     ## Now test the "SEND TEST NOTIFICATION" is working
     os.unlink("test-datastore/notification.txt")
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.post(url_for("ajax_callback_send_notification_test", watch_uuid=uuid), data={}, follow_redirects=True)
     time.sleep(5)
     assert os.path.isfile("test-datastore/notification.txt"), "Notification received"

changedetectionio/tests/test_rss.py

@@ -132,7 +132,7 @@ def test_rss_xpath_filtering(client, live_server, measure_memory_usage):
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
         data={

changedetectionio/tests/test_scheduler.py

@@ -39,7 +39,7 @@ def test_check_basic_scheduler_functionality(client, live_server, measure_memory
 
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
 
@@ -104,7 +104,7 @@ def test_check_basic_global_scheduler_functionality(client, live_server, measure
 
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
 

changedetectionio/tests/test_whois_processor.py

@@ -1,182 +0,0 @@
-import pytest
-from unittest.mock import MagicMock, patch
-from changedetectionio.processors.whois_plugin import WhoisProcessor
-
-
-class MockWatch:
-    def __init__(self, url, previous_md5=None, include_filters=None, ignore_text=None):
-        self.url = url
-        self._previous_md5 = previous_md5
-        self._include_filters = include_filters or []
-        self._ignore_text = ignore_text or []
-        self.history = {}
-    
-    def get(self, key, default=None):
-        if key == 'previous_md5':
-            return self._previous_md5
-        elif key == 'include_filters':
-            return self._include_filters
-        elif key == 'ignore_text':
-            return self._ignore_text
-        elif key == 'url':
-            return self.url
-        return default
-    
-    def has_special_diff_filter_options_set(self):
-        return False
-
-
-@patch('whois.whois')
-@patch('changedetectionio.processors.difference_detection_processor.__init__')
-@patch('changedetectionio.processors.text_json_diff.processor.perform_site_check.run_changedetection')
-def test_whois_processor_basic_functionality(mock_super_run, mock_base_init, mock_whois):
-    """Test the basic functionality of the WhoisProcessor"""
-    # Mock the base class init so we don't need to set up the full watch structure
-    mock_base_init.return_value = None
-    
-    # Mock super().run_changedetection to return a simple result
-    mock_super_run.return_value = (False, {'previous_md5': 'some-md5'}, b'Some filtered text')
-    
-    # Mock the whois response
-    mock_whois_result = MagicMock()
-    mock_whois_result.text = "Domain Name: example.com\nRegistrar: Example Registrar\nCreation Date: 2020-01-01\n"
-    mock_whois.return_value = mock_whois_result
-    
-    # Create mock datastore
-    mock_datastore = MagicMock()
-    mock_datastore.proxy_list = None  # No proxies
-    mock_datastore.get_preferred_proxy_for_watch.return_value = None
-    mock_datastore.data = {
-        'settings': {
-            'application': {
-                'allow_file_uri': False
-            }
-        }
-    }
-    
-    # Create a processor instance and setup minimal required attributes
-    processor = WhoisProcessor(datastore=mock_datastore, watch_uuid='test-uuid')
-    
-    # Create a minimal watch object
-    watch = MockWatch(url="https://example.com")
-    
-    # Simulate link access in the watch
-    processor.watch = MagicMock()
-    processor.watch.link = "https://example.com"
-    processor.watch.get.return_value = "uuid-123"
-    
-    # Run the processor's run_changedetection method by first using call_browser
-    processor.call_browser()
-    
-    # Check that the fetcher was set up correctly
-    assert processor.fetcher is not None
-    assert hasattr(processor.fetcher, 'content')
-    assert hasattr(processor.fetcher, 'headers')
-    assert hasattr(processor.fetcher, 'status_code')
-    
-    # Verify that whois was called with the right domain
-    assert mock_whois.called
-    assert mock_whois.call_args[0][0] == 'example.com'
-    
-    # Now run the processor
-    result = processor.run_changedetection(watch)
-    
-    # Check that the parent run_changedetection was called
-    assert mock_super_run.called
-
-
-@patch('whois.whois')
-@patch('changedetectionio.processors.difference_detection_processor.__init__')
-def test_whois_processor_call_browser_with_proxy(mock_base_init, mock_whois):
-    """Test the call_browser method with proxy configuration"""
-    # Mock the base class init
-    mock_base_init.return_value = None
-    
-    # Mock the whois response
-    mock_whois_result = MagicMock()
-    mock_whois_result.text = "Domain Name: example.com\nRegistrar: Example Registrar\nCreation Date: 2020-01-01\n"
-    mock_whois.return_value = mock_whois_result
-    
-    # Create mock datastore
-    mock_datastore = MagicMock()
-    mock_proxy = {
-        'test-proxy': {
-            'url': 'http://proxy.example.com:8080',
-            'label': 'Test Proxy'
-        }
-    }
-    mock_datastore.proxy_list = mock_proxy
-    mock_datastore.get_preferred_proxy_for_watch.return_value = 'test-proxy'
-    mock_datastore.data = {
-        'settings': {
-            'application': {
-                'allow_file_uri': False
-            }
-        }
-    }
-    
-    # Create a processor instance with our mock datastore
-    processor = WhoisProcessor(datastore=mock_datastore, watch_uuid='test-uuid')
-    
-    # Set up watch
-    processor.watch = MagicMock()
-    processor.watch.link = "https://example.com"
-    processor.watch.get.return_value = "uuid-123"
-    
-    # Call the method with a proxy
-    processor.call_browser()
-    
-    # Verify whois was called
-    assert mock_whois.called
-    assert mock_whois.call_args[0][0] == 'example.com'
-    
-    # Check that the fetcher was set up correctly
-    assert processor.fetcher is not None
-    assert processor.fetcher.content is not None
-
-
-@patch('changedetectionio.processors.difference_detection_processor.__init__')
-def test_whois_processor_perform_site_check(mock_base_init):
-    """Test the WhoisProcessor.perform_site_check static method"""
-    mock_base_init.return_value = None
-    
-    # Test the static method
-    with patch.object(WhoisProcessor, '__init__', return_value=None) as mock_init:
-        datastore = MagicMock()
-        watch_uuid = "test-uuid"
-        
-        # Call the static method
-        processor = WhoisProcessor.perform_site_check(datastore=datastore, watch_uuid=watch_uuid)
-        
-        # Check that constructor was called with expected args
-        mock_init.assert_called_once_with(datastore=datastore, watch_uuid=watch_uuid)
-        
-        # Check it returns the right type
-        assert isinstance(processor, WhoisProcessor)
-
-
-def test_get_display_link():
-    """Test the get_display_link hook implementation"""
-    from changedetectionio.processors.whois_plugin import get_display_link
-    
-    # Test with a regular URL
-    url = "https://example.com/some/path?param=value"
-    processor_name = "whois"
-    link = get_display_link(url=url, processor_name=processor_name)
-    assert link == "WHOIS - example.com"
-    
-    # Test with a subdomain
-    url = "https://subdomain.example.com/"
-    link = get_display_link(url=url, processor_name=processor_name)
-    assert link == "WHOIS - subdomain.example.com"
-    
-    # Test with www prefix (should be removed)
-    url = "https://www.example.com/"
-    link = get_display_link(url=url, processor_name=processor_name)
-    assert link == "WHOIS - example.com"
-    
-    # Test with a different processor (should return None)
-    url = "https://example.com/"
-    processor_name = "text_json_diff"
-    link = get_display_link(url=url, processor_name=processor_name)
-    assert link is None

changedetectionio/tests/unit/test_conditions.py

@@ -1,82 +0,0 @@
-from changedetectionio.conditions import execute_ruleset_against_all_plugins
-from changedetectionio.store import ChangeDetectionStore
-import shutil
-import tempfile
-import time
-import unittest
-import uuid
-
-
-class TestTriggerConditions(unittest.TestCase):
-    def setUp(self):
-
-        # Create a temporary directory for the test datastore
-        self.test_datastore_path = tempfile.mkdtemp()
-
-        # Initialize ChangeDetectionStore with our test path and no default watches
-        self.store = ChangeDetectionStore(
-            datastore_path=self.test_datastore_path,
-            include_default_watches=False
-        )
-
-        # Add a test watch
-        watch_url = "https://example.com"
-        self.watch_uuid = self.store.add_watch(url=watch_url)
-
-    def tearDown(self):
-      # Clean up the test datastore
-      self.store.stop_thread = True
-      time.sleep(0.5)  # Give thread time to stop
-      shutil.rmtree(self.test_datastore_path)
-
-    def test_conditions_execution_pass(self):
-        # Get the watch object
-        watch = self.store.data['watching'][self.watch_uuid]
-
-        # Create and save a snapshot
-        first_content = "I saw 100 people at a rock show"
-        timestamp1 = int(time.time())
-        snapshot_id1 = str(uuid.uuid4())
-        watch.save_history_text(contents=first_content,
-                                timestamp=timestamp1,
-                                snapshot_id=snapshot_id1)
-
-        # Add another snapshot
-        second_content = "I saw 200 people at a rock show"
-        timestamp2 = int(time.time()) + 60
-        snapshot_id2 = str(uuid.uuid4())
-        watch.save_history_text(contents=second_content,
-                                timestamp=timestamp2,
-                                snapshot_id=snapshot_id2)
-
-        # Verify both snapshots are stored
-        history = watch.history
-        self.assertEqual(len(history), 2)
-
-        # Retrieve and check snapshots
-        #snapshot1 = watch.get_history_snapshot(str(timestamp1))
-        #snapshot2 = watch.get_history_snapshot(str(timestamp2))
-
-        self.store.data['watching'][self.watch_uuid].update(
-            {
-                "conditions_match_logic": "ALL",
-                "conditions": [
-                    {"operator": ">=", "field": "extracted_number", "value": "10"},
-                    {"operator": "<=", "field": "extracted_number", "value": "5000"},
-                    {"operator": "in", "field": "page_text", "value": "rock"},
-                    #{"operator": "starts_with", "field": "page_text", "value": "I saw"},
-                ]
-            }
-        )
-
-        # ephemeral_data - some data that could exist before the watch saved a new version
-        result = execute_ruleset_against_all_plugins(current_watch_uuid=self.watch_uuid,
-                                                     application_datastruct=self.store.data,
-                                                     ephemeral_data={'text': "I saw 500 people at a rock show"})
-
-        # @todo - now we can test that 'Extract number' increased more than X since last time
-        self.assertTrue(result)
-
-
-if __name__ == '__main__':
-    unittest.main()

changedetectionio/tests/unit/test_semver.py

@@ -1,64 +0,0 @@
-#!/usr/bin/env python3
-
-# run from dir above changedetectionio/ dir
-# python3 -m unittest changedetectionio.tests.unit.test_semver
-
-import re
-import unittest
-
-
-# The SEMVER regex
-SEMVER_REGEX = r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
-
-# Compile the regex
-semver_pattern = re.compile(SEMVER_REGEX)
-
-class TestSemver(unittest.TestCase):
-    def test_valid_versions(self):
-        """Test valid semantic version strings"""
-        valid_versions = [
-            "1.0.0",
-            "0.1.0",
-            "0.0.1",
-            "1.0.0-alpha",
-            "1.0.0-alpha.1",
-            "1.0.0-0.3.7",
-            "1.0.0-x.7.z.92",
-            "1.0.0-alpha+001",
-            "1.0.0+20130313144700",
-            "1.0.0-beta+exp.sha.5114f85"
-        ]
-        for version in valid_versions:
-            with self.subTest(version=version):
-                self.assertIsNotNone(semver_pattern.match(version), f"Version {version} should be valid")
-
-    def test_invalid_versions(self):
-        """Test invalid semantic version strings"""
-        invalid_versions = [
-            "0.48.06",
-            "1.0",
-            "1.0.0-",
-# Seems to pass the semver.org regex?
-#            "1.0.0-alpha-",
-            "1.0.0+",
-            "1.0.0-alpha+",
-            "1.0.0-",
-            "01.0.0",
-            "1.01.0",
-            "1.0.01",
-            ".1.0.0",
-            "1..0.0"
-        ]
-        for version in invalid_versions:
-            with self.subTest(version=version):
-                res = semver_pattern.match(version)
-                self.assertIsNone(res, f"Version '{version}' should be invalid")
-
-    def test_our_version(self):
-        from changedetectionio import get_version
-        our_version = get_version()
-        self.assertIsNotNone(semver_pattern.match(our_version), f"Our version '{our_version}' should be a valid SEMVER string")
-
-
-if __name__ == '__main__':
-    unittest.main()

changedetectionio/tests/unit/test_watch_model.py

@@ -16,6 +16,7 @@ class TestDiffBuilder(unittest.TestCase):
         watch = Watch.model(datastore_path='/tmp', default={})
         watch.ensure_data_dir_exists()
 
+        watch['last_viewed'] = 110
 
         # Contents from the browser are always returned from the browser/requests/etc as str, str is basically UTF-16 in python
         watch.save_history_text(contents="hello world", timestamp=100, snapshot_id=str(uuid_builder.uuid4()))
@@ -24,59 +25,31 @@ class TestDiffBuilder(unittest.TestCase):
         watch.save_history_text(contents="hello world", timestamp=112, snapshot_id=str(uuid_builder.uuid4()))
         watch.save_history_text(contents="hello world", timestamp=115, snapshot_id=str(uuid_builder.uuid4()))
         watch.save_history_text(contents="hello world", timestamp=117, snapshot_id=str(uuid_builder.uuid4()))
-    
-        p = watch.get_from_version_based_on_last_viewed
-        assert p == "100", "Correct 'last viewed' timestamp was detected"
 
-        watch['last_viewed'] = 110
-        p = watch.get_from_version_based_on_last_viewed
-        assert p == "109", "Correct 'last viewed' timestamp was detected"
+        p = watch.get_next_snapshot_key_to_last_viewed
+        assert p == "112", "Correct last-viewed timestamp was detected"
 
+        # When there is only one step of difference from the end of the list, it should return second-last change
         watch['last_viewed'] = 116
-        p = watch.get_from_version_based_on_last_viewed
-        assert p == "115", "Correct 'last viewed' timestamp was detected"
+        p = watch.get_next_snapshot_key_to_last_viewed
+        assert p == "115", "Correct 'second last' last-viewed timestamp was detected when using the last timestamp"
 
         watch['last_viewed'] = 99
-        p = watch.get_from_version_based_on_last_viewed
-        assert p == "100", "When the 'last viewed' timestamp is less than the oldest snapshot, return oldest"
+        p = watch.get_next_snapshot_key_to_last_viewed
+        assert p == "100"
 
         watch['last_viewed'] = 200
-        p = watch.get_from_version_based_on_last_viewed
-        assert p == "115", "When the 'last viewed' timestamp is greater than the newest snapshot, return second newest"
+        p = watch.get_next_snapshot_key_to_last_viewed
+        assert p == "115", "When the 'last viewed' timestamp is greater than the newest snapshot, return second last "
 
         watch['last_viewed'] = 109
-        p = watch.get_from_version_based_on_last_viewed
+        p = watch.get_next_snapshot_key_to_last_viewed
         assert p == "109", "Correct when its the same time"
 
         # new empty one
         watch = Watch.model(datastore_path='/tmp', default={})
-        p = watch.get_from_version_based_on_last_viewed
+        p = watch.get_next_snapshot_key_to_last_viewed
         assert p == None, "None when no history available"
 
-        watch.save_history_text(contents="hello world", timestamp=100, snapshot_id=str(uuid_builder.uuid4()))
-        p = watch.get_from_version_based_on_last_viewed
-        assert p == "100", "Correct with only one history snapshot"
-
-        watch['last_viewed'] = 200
-        p = watch.get_from_version_based_on_last_viewed
-        assert p == "100", "Correct with only one history snapshot"
-
-    def test_watch_link_property_with_processor(self):
-        """Test the link property with a processor that customizes the link"""
-        from unittest.mock import patch
-        
-        watch = Watch.model(datastore_path='/tmp', default={})
-        watch['url'] = 'https://example.com'
-        watch['processor'] = 'whois'
-        
-        # Mock the processor registry's get_display_link function
-        with patch('changedetectionio.processors.processor_registry.get_display_link') as mock_get_display_link:
-            mock_get_display_link.return_value = "WHOIS - example.com"
-            
-            # The link property should use the customized link from the processor
-            assert watch.link == "WHOIS - example.com"
-            mock_get_display_link.assert_called_once_with(url='https://example.com', processor_name='whois')
-
-
 if __name__ == '__main__':
     unittest.main()

changedetectionio/tests/util.py

@@ -76,14 +76,6 @@ def set_more_modified_response():
     return None
 
 
-def set_empty_text_response():
-    test_return_data = """<html><body></body></html>"""
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-    return None
-
 def wait_for_notification_endpoint_output():
     '''Apprise can take a few seconds to fire'''
     #@todo - could check the apprise object directly instead of looking for this file
@@ -223,10 +215,9 @@ def live_server_setup(live_server):
     def test_method():
         return request.method
 
-    # Where we POST to as a notification, also use a space here to test URL escaping is OK across all tests that use this. ( #2868 )
-    @live_server.app.route('/test_notification endpoint', methods=['POST', 'GET'])
+    # Where we POST to as a notification
+    @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET'])
     def test_notification_endpoint():
-
         with open("test-datastore/notification.txt", "wb") as f:
             # Debug method, dump all POST to file also, used to prove #65
             data = request.stream.read()
@@ -244,11 +235,8 @@ def live_server_setup(live_server):
                 f.write(request.content_type)
 
         print("\n>> Test notification endpoint was hit.\n", data)
+        return "Text was set"
 
-        content = "Text was set"
-        status_code = request.args.get('status_code',200)
-        resp = make_response(content, status_code)
-        return resp
 
     # Just return the verb in the request
     @live_server.app.route('/test-basicauth', methods=['GET'])
@@ -285,43 +273,15 @@ def live_server_setup(live_server):
             <p id="remove">This text should be removed</p>
               <form onsubmit="event.preventDefault();">
             <!-- obfuscated text so that we dont accidentally get a false positive due to conversion of the source :) --->
-                <button name="test-button" onclick="
-                getElementById('remove').remove();
-                getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=');
-                getElementById('reflect-text').innerHTML = getElementById('test-input-text').value;
-                ">Click here</button>
-                
-                <div id="some-content"></div>
-                
+                <button name="test-button" onclick="getElementById('remove').remove();getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=')">Click here</button>
+                <div id=some-content></div>
                 <pre>
                 {header_text.lower()}
                 </pre>
-                
-                <br>
-                <!-- used for testing that the jinja2 compiled here --->
-                <input type="text" value="" id="test-input-text" /><br>
-                <div id="reflect-text">Waiting to reflect text from #test-input-text here</div>
-              </form>
-                
-           </body>
+              </body>
          </html>""", 200)
         resp.headers['Content-Type'] = 'text/html'
         return resp
 
     live_server.start()
 
-def get_index(client):
-    import inspect
-    # Get the caller's frame (parent function)
-    frame = inspect.currentframe()
-    caller_frame = frame.f_back  # Go back to the caller's frame
-    caller_name = caller_frame.f_code.co_name
-    caller_line = caller_frame.f_lineno
-
-    print(f"Called by: {caller_name}, Line: {caller_line}")
-
-    res = client.get(url_for("index"))
-    with open(f"test-datastore/index-{caller_name}-{caller_line}.html", 'wb') as f:
-        f.write(res.data)
-
-    return res

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -2,16 +2,14 @@
 
 import os
 from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks, get_index
+from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
 
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
 def test_visual_selector_content_ready(client, live_server, measure_memory_usage):
-    live_server.stop()
-    live_server.start()
 
     import os
     import json
@@ -29,7 +27,7 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
         data={
@@ -89,9 +87,7 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
 
 def test_basic_browserstep(client, live_server, measure_memory_usage):
 
-    live_server.stop()
-    live_server.start()
-
+    #live_server_setup(live_server)
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
 
     test_url = url_for('test_interactive_html_endpoint', _external=True)
@@ -112,13 +108,9 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
             "url": test_url,
             "tags": "",
             'fetch_backend': "html_webdriver",
-            'browser_steps-0-operation': 'Enter text in field',
-            'browser_steps-0-selector': '#test-input-text',
-            # Should get set to the actual text (jinja2 rendered)
-            'browser_steps-0-optional_value': "Hello-Jinja2-{% now  'Europe/Berlin', '%Y-%m-%d' %}",
-            'browser_steps-1-operation': 'Click element',
-            'browser_steps-1-selector': 'button[name=test-button]',
-            'browser_steps-1-optional_value': '',
+            'browser_steps-0-operation': 'Click element',
+            'browser_steps-0-selector': 'button[name=test-button]',
+            'browser_steps-0-optional_value': '',
             # For now, cookies doesnt work in headers because it must be a full cookiejar object
             'headers': "testheader: yes\buser-agent: MyCustomAgent",
         },
@@ -127,7 +119,7 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
 
     assert b"This text should be removed" not in res.data
@@ -140,32 +132,13 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
     assert b"This text should be removed" not in res.data
     assert b"I smell JavaScript because the button was pressed" in res.data
 
-    assert b'Hello-Jinja2-20' in res.data
-
     assert b"testheader: yes" in res.data
     assert b"user-agent: mycustomagent" in res.data
-    live_server.stop()
-
-def test_non_200_errors_report_browsersteps(client, live_server):
-
-    live_server.stop()
-    live_server.start()
 
     four_o_four_url =  url_for('test_endpoint', status_code=404, _external=True)
     four_o_four_url = four_o_four_url.replace('localhost.localdomain', 'cdio')
     four_o_four_url = four_o_four_url.replace('localhost', 'cdio')
 
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": four_o_four_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added in Paused state, saving will unpause" in res.data
-    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
-
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-
     # now test for 404 errors
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
@@ -180,14 +153,12 @@ def test_non_200_errors_report_browsersteps(client, live_server):
         follow_redirects=True
     )
     assert b"unpaused" in res.data
-
     wait_for_all_checks(client)
 
-    res = get_index(client)
-
+    res = client.get(url_for("index"))
     assert b'Error - 404' in res.data
 
     client.get(
         url_for("form_delete", uuid="all"),
         follow_redirects=True
-    )
+    )

changedetectionio/update_worker.py

@@ -243,6 +243,7 @@ class update_worker(threading.Thread):
                 os.unlink(full_path)
 
     def run(self):
+        now = time.time()
         
         while not self.app.config.exit.is_set():
             update_handler = None
@@ -253,7 +254,6 @@ class update_worker(threading.Thread):
                 pass
 
             else:
-                fetch_start_time = time.time()
                 uuid = queued_item_data.item.get('uuid')
                 self.current_uuid = uuid
                 if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
@@ -268,48 +268,26 @@ class update_worker(threading.Thread):
                     watch = self.datastore.data['watching'].get(uuid)
 
                     logger.info(f"Processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
+                    now = time.time()
 
                     try:
                         # Processor is what we are using for detecting the "Change"
-                        processor_name = watch.get('processor', 'text_json_diff')
+                        processor = watch.get('processor', 'text_json_diff')
 
-                        
-
-                        # First, try to get the processor from our plugin registry
+                        # Init a new 'difference_detection_processor', first look in processors
+                        processor_module_name = f"changedetectionio.processors.{processor}.processor"
                         try:
-                            from changedetectionio.processors.processor_registry import get_processor_site_check
-                            update_handler = get_processor_site_check(processor_name, self.datastore, uuid)
-                            
-                            if update_handler:
-                                # We found the processor in our plugin registry
-                                logger.info(f"Using processor '{processor_name}' from plugin registry")
-                            else:
-                                # Fall back to the traditional file-based approach
-                                processor_module_name = f"changedetectionio.processors.{processor_name}.processor"
-                                try:
-                                    processor_module = importlib.import_module(processor_module_name)
-                                    update_handler = processor_module.perform_site_check(datastore=self.datastore,
-                                                                                        watch_uuid=uuid)
-                                except ModuleNotFoundError as e:
-                                    print(f"Processor module '{processor_name}' not found in both plugin registry and file system.")
-                                    raise e
-                        except ImportError as e:
-                            # If processor_registry.py cannot be imported, fall back to the traditional approach
-                            processor_module_name = f"changedetectionio.processors.{processor_name}.processor"
-                            try:
-                                processor_module = importlib.import_module(processor_module_name)
-                                update_handler = processor_module.perform_site_check(datastore=self.datastore,
-                                                                                    watch_uuid=uuid)
-                            except ModuleNotFoundError as e:
-                                print(f"Processor module '{processor_name}' not found.")
-                                raise e
+                            processor_module = importlib.import_module(processor_module_name)
+                        except ModuleNotFoundError as e:
+                            print(f"Processor module '{processor}' not found.")
+                            raise e
+
+                        update_handler = processor_module.perform_site_check(datastore=self.datastore,
+                                                                             watch_uuid=uuid
+                                                                             )
 
                         update_handler.call_browser()
 
-                        # In reality, the actual time of when the change was detected could be a few seconds after this
-                        # For example it should include when the page stopped rendering if using a playwright/chrome type fetch
-                        fetch_start_time = time.time()
-
                         changed_detected, update_obj, contents = update_handler.run_changedetection(watch=watch)
 
                         # Re #342
@@ -534,7 +512,7 @@ class update_worker(threading.Thread):
 
                     if not self.datastore.data['watching'].get(uuid):
                         continue
-
+                    #
                     # Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
                     if process_changedetection_results:
 
@@ -547,6 +525,8 @@ class update_worker(threading.Thread):
                                 except Exception as e:
                                     logger.warning(f"UUID: {uuid} Extract <title> as watch title was enabled, but couldn't find a <title>.")
 
+                        # Now update after running everything
+                        timestamp = round(time.time())
                         try:
                             self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
 
@@ -562,28 +542,24 @@ class update_worker(threading.Thread):
 
                                 # Small hack so that we sleep just enough to allow 1 second  between history snapshots
                                 # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
-                                # @also - the keys are one per second at the most (for now)
-                                if watch.newest_history_key and int(fetch_start_time) == int(watch.newest_history_key):
+
+                                if watch.newest_history_key and int(timestamp) == int(watch.newest_history_key):
                                     logger.warning(
-                                        f"Timestamp {fetch_start_time} already exists, waiting 1 seconds so we have a unique key in history.txt")
-                                    fetch_start_time += 1
+                                        f"Timestamp {timestamp} already exists, waiting 1 seconds so we have a unique key in history.txt")
+                                    timestamp = str(int(timestamp) + 1)
                                     time.sleep(1)
 
                                 watch.save_history_text(contents=contents,
-                                                        timestamp=int(fetch_start_time),
+                                                        timestamp=timestamp,
                                                         snapshot_id=update_obj.get('previous_md5', 'none'))
 
-
-                                empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
-                                if update_handler.fetcher.content or (not update_handler.fetcher.content and empty_pages_are_a_change):
-                                    # attribute .last_changed is then based on this data
-                                    watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=int(fetch_start_time))
+                                if update_handler.fetcher.content:
+                                    watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=timestamp)
 
                                 # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
                                 if watch.history_n >= 2:
                                     logger.info(f"Change detected in UUID {uuid} - {watch['url']}")
                                     if not watch.get('notification_muted'):
-                                        # @todo only run this if notifications exist
                                         self.send_content_changed_notification(watch_uuid=uuid)
 
                         except Exception as e:
@@ -605,15 +581,15 @@ class update_worker(threading.Thread):
                     except Exception as e:
                         pass
 
-                    self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - fetch_start_time, 3),
-                                                                       'last_checked': int(fetch_start_time),
+                    self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
+                                                                       'last_checked': round(time.time()),
                                                                        'check_count': count
                                                                        })
 
 
                 self.current_uuid = None  # Done
                 self.q.task_done()
-                logger.debug(f"Watch {uuid} done in {time.time()-fetch_start_time:.2f}s")
+                logger.debug(f"Watch {uuid} done in {time.time()-now:.2f}s")
 
                 # Give the CPU time to interrupt
                 time.sleep(0.1)

docker-compose.yml

@@ -1,3 +1,4 @@
+version: '3.2'
 services:
     changedetection:
       image: ghcr.io/dgtlmoon/changedetection.io
@@ -11,6 +12,9 @@ services:
   #    environment:
   #        Default listening port, can also be changed with the -p option
   #      - PORT=5000
+
+  #      - PUID=1000
+  #      - PGID=1000
   #
   #        Log levels are in descending order. (TRACE is the most detailed one)
   #        Log output levels: TRACE, DEBUG(default), INFO, SUCCESS, WARNING, ERROR, CRITICAL
@@ -81,7 +85,7 @@ services:
 
 
      # Sockpuppetbrowser is basically chrome wrapped in an API for allowing fast fetching of web-pages.
-     # RECOMMENDED FOR FETCHING PAGES WITH CHROME, be sure to enable the "PLAYWRIGHT_DRIVER_URL" env variable in the main changedetection container
+     # RECOMMENDED FOR FETCHING PAGES WITH CHROME
 #    sockpuppetbrowser:
 #        hostname: sockpuppetbrowser
 #        image: dgtlmoon/sockpuppetbrowser:latest

requirements.txt

@@ -35,7 +35,7 @@ dnspython==2.6.1 # related to eventlet fixes
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise==1.9.2
+apprise==1.9.0
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 # use any version other than 2.0.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
@@ -95,20 +95,5 @@ babel
 # Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096
 greenlet >= 3.0.3
 
-# Pinned or it causes problems with flask_expects_json which seems unmaintained
-referencing==0.35.1
-
-# For conditions
-panzi-json-logic
-# For conditions - extracted number from a body of text
-price-parser
-
 # Scheduler - Windows seemed to miss a lot of default timezone info (even "UTC" !)
 tzdata
-
-#typing_extensions ==4.8.0
-
-pluggy ~= 1.5
-
-
-

test_processor_registration.py

@@ -1,31 +0,0 @@
-#!/usr/bin/env python3
-
-from changedetectionio.processors import available_processors
-from changedetectionio.processors.processor_registry import get_processor_class, get_processor_form
-
-# Test processor registration
-print("=== Available Processors ===")
-processors = available_processors()
-for name, description in processors:
-    print(f"Processor: {name} - {description}")
-
-# Check if our WHOIS processor is registered
-whois_processor_name = "whois_processor"
-whois_found = any(name == whois_processor_name for name, _ in processors)
-
-if whois_found:
-    print(f"\nWHOIS Processor found! Getting processor class and form...")
-    
-    # Get the processor class
-    processor_class = get_processor_class(whois_processor_name)
-    print(f"Processor class: {processor_class}")
-    print(f"Processor class name: {processor_class.__name__ if processor_class else None}")
-    print(f"Processor class module: {processor_class.__module__ if processor_class else None}")
-    
-    # Get the processor form
-    processor_form = get_processor_form(whois_processor_name)
-    print(f"Processor form: {processor_form}")
-    
-    print("\nWHOIS Processor successfully registered")
-else:
-    print(f"\nWHOIS Processor not found in available processors")

test_processors.py

@@ -1,16 +0,0 @@
-#!/usr/bin/env python3
-
-from changedetectionio.processors import available_processors
-from changedetectionio.processors import find_processors
-
-# Test traditional processor discovery
-print("=== Traditional Processor Discovery ===")
-traditional_processors = find_processors()
-for module, name in traditional_processors:
-    print(f"Found processor: {name} in {module.__name__}")
-
-# Test combined processor discovery (traditional + pluggy)
-print("\n=== Combined Processor Discovery ===")
-combined_processors = available_processors()
-for name, description in combined_processors:
-    print(f"Processor: {name} - {description}")

test_whois_extraction.py

@@ -1,53 +0,0 @@
-#!/usr/bin/env python3
-
-import urllib.parse
-import re
-import sys
-
-def extract_domain_from_url(url):
-    """Extract domain from a URL"""
-    parsed_url = urllib.parse.urlparse(url)
-    domain = parsed_url.netloc
-    
-    # Remove www. prefix if present
-    domain = re.sub(r'^www\.', '', domain)
-    
-    return domain
-
-# Test domain extraction
-test_urls = [
-    "https://changedetection.io",
-    "http://www.example.com/page",
-    "https://subdomain.domain.co.uk/path?query=1",
-    "ftp://ftp.example.org",
-    "https://www.changedetection.io/page/subpage",
-]
-
-print("=== Domain Extraction Test ===")
-for url in test_urls:
-    domain = extract_domain_from_url(url)
-    print(f"URL: {url} -> Domain: {domain}")
-
-# Test WHOIS lookup for changedetection.io
-try:
-    import whois
-    
-    domain = extract_domain_from_url("https://changedetection.io")
-    print(f"\n=== WHOIS lookup for {domain} ===")
-    
-    whois_info = whois.whois(domain)
-    
-    # Print key information
-    print(f"Domain Name: {whois_info.get('domain_name', '')}")
-    print(f"Registrar: {whois_info.get('registrar', '')}")
-    print(f"Creation Date: {whois_info.get('creation_date', '')}")
-    print(f"Expiration Date: {whois_info.get('expiration_date', '')}")
-    
-    print("\nWHOIS lookup successful!")
-    
-except ImportError:
-    print("python-whois module not installed. Run: pip install python-whois")
-    sys.exit(1)
-except Exception as e:
-    print(f"Error performing WHOIS lookup: {str(e)}")
-    sys.exit(1)

test_whois_processor.py

@@ -1,47 +0,0 @@
-#!/usr/bin/env python3
-
-from changedetectionio.processors import available_processors
-from changedetectionio.processors.processor_registry import get_processor_class
-import urllib.parse
-import sys
-
-# First, verify our processor is available
-print("=== Available Processors ===")
-processors = available_processors()
-for name, description in processors:
-    print(f"Processor: {name} - {description}")
-
-# Get the WHOIS processor class
-whois_processor_class = get_processor_class("whois_processor")
-if not whois_processor_class:
-    print("ERROR: WHOIS processor not found in available processors.")
-    sys.exit(1)
-
-print(f"\nFound WHOIS processor class: {whois_processor_class}")
-
-# Test the WHOIS processor directly
-try:
-    # Parse a domain from a URL
-    url = "https://changedetection.io"
-    parsed_url = urllib.parse.urlparse(url)
-    domain = parsed_url.netloc
-    
-    # Import whois and fetch information
-    import whois
-    whois_info = whois.whois(domain)
-    
-    print(f"\n=== WHOIS Information for {domain} ===")
-    
-    # Print the information
-    if hasattr(whois_info, 'text'):
-        print(whois_info.text)
-    else:
-        for key, value in whois_info.items():
-            if value:
-                print(f"{key}: {value}")
-                
-    print("\nSuccessfully retrieved WHOIS data!")
-    
-except Exception as e:
-    print(f"Error fetching WHOIS data: {str(e)}")
-    sys.exit(1)

test_whois_processor_full.py

@@ -1,136 +0,0 @@
-#!/usr/bin/env python3
-
-from changedetectionio.processors import available_processors
-from changedetectionio.processors.processor_registry import get_processor_class
-import unittest
-import sys
-from unittest.mock import MagicMock, patch
-import urllib.parse
-
-# First, verify our processor is available
-print("=== Available Processors ===")
-processors = available_processors()
-for name, description in processors:
-    print(f"Processor: {name} - {description}")
-
-# Get the WHOIS processor class
-whois_processor_class = get_processor_class("whois_processor")
-if not whois_processor_class:
-    print("ERROR: WHOIS processor not found in available processors.")
-    sys.exit(1)
-
-print(f"\nFound WHOIS processor class: {whois_processor_class}")
-
-# Create a test for our WHOIS processor
-class TestWhoisProcessor(unittest.TestCase):
-    
-    # Use the real whois function - tests will actually make network requests
-    def test_whois_processor_real(self):
-        # Extract the domain from the URL
-        test_url = "https://changedetection.io"
-        parsed_url = urllib.parse.urlparse(test_url)
-        domain = parsed_url.netloc
-        
-        # Create a minimal mock datastore
-        mock_datastore = MagicMock()
-        mock_datastore.data = {
-            'watching': {'test-uuid': {'url': test_url}},
-            'settings': {
-                'application': {'empty_pages_are_a_change': False},
-                'requests': {'timeout': 30}
-            }
-        }
-        mock_datastore.get_all_base_headers.return_value = {}
-        mock_datastore.get_all_headers_in_textfile_for_watch.return_value = {}
-        mock_datastore.get_preferred_proxy_for_watch.return_value = None
-        mock_datastore.get_tag_overrides_for_watch.return_value = []
-        
-        # Create a minimal mock watch that mimics the real Watch class
-        class MockWatch:
-            def __init__(self, url):
-                self.link = url
-                self.is_pdf = False
-                self.has_browser_steps = False
-                self.is_source_type_url = False
-                self.history = {}
-                self.history_n = 0
-                self.last_viewed = 0
-                self.newest_history_key = 0
-                
-            def get(self, key, default=None):
-                if key == 'uuid':
-                    return 'test-uuid'
-                elif key == 'include_filters':
-                    return []
-                elif key == 'body':
-                    return None
-                elif key == 'method':
-                    return 'GET'
-                elif key == 'headers':
-                    return {}
-                elif key == 'browser_steps':
-                    return []
-                return default
-                
-            def __getitem__(self, key):
-                return self.get(key)
-                
-            def get_last_fetched_text_before_filters(self):
-                return ""
-            
-            def save_last_text_fetched_before_filters(self, content):
-                pass
-                
-            def has_special_diff_filter_options_set(self):
-                return False
-                
-            def lines_contain_something_unique_compared_to_history(self, lines, ignore_whitespace):
-                return True
-                
-        mock_watch = MockWatch(test_url)
-        
-        # Create a more complete mock fetcher
-        class MockFetcher:
-            def __init__(self):
-                self.content = ""
-                self.raw_content = b""
-                self.headers = {'Content-Type': 'text/plain'}
-                self.screenshot = None
-                self.xpath_data = None
-                self.instock_data = None
-                self.browser_steps = []
-            
-            def get_last_status_code(self):
-                return 200
-                
-            def get_all_headers(self):
-                return {'content-type': 'text/plain'}
-                
-            def quit(self):
-                pass
-                
-            def run(self, **kwargs):
-                pass
-                
-        # Create the processor and set the mock fetcher
-        processor = whois_processor_class(datastore=mock_datastore, watch_uuid='test-uuid')
-        processor.fetcher = MockFetcher()
-        
-        # Run the processor - this will make an actual WHOIS request
-        changed, update_obj, content = processor.run_changedetection(mock_watch)
-        
-        # Print the content for debugging
-        content_str = content.decode('utf-8')
-        print(f"\n=== WHOIS Content from processor (first 200 chars) ===")
-        print(content_str[:200] + "...")
-        
-        # Verify the content contains domain information
-        self.assertIn(domain, content_str)
-        self.assertIn("Domain Name", content_str)
-        self.assertIn("Creation Date", content_str)
-        
-        print("\nWHOIS processor test with real data PASSED!")
-
-# Run the test
-if __name__ == "__main__":
-    unittest.main(argv=['first-arg-is-ignored'], exit=False)

test_whois_simple.py

@@ -1,39 +0,0 @@
-#!/usr/bin/env python3
-
-import urllib.parse
-import re
-import whois
-
-# Test with changedetection.io domain
-url = "https://changedetection.io"
-
-# Extract domain from URL
-parsed_url = urllib.parse.urlparse(url)
-domain = parsed_url.netloc
-
-# Remove www. prefix if present
-domain = re.sub(r'^www\.', '', domain)
-
-# Fetch WHOIS information
-print(f"Looking up WHOIS data for domain: {domain}")
-whois_info = whois.whois(domain)
-
-# Print key WHOIS data
-print("\nKey WHOIS information:")
-print(f"Domain Name: {whois_info.get('domain_name', 'Unknown')}")
-print(f"Registrar: {whois_info.get('registrar', 'Unknown')}")
-print(f"Creation Date: {whois_info.get('creation_date', 'Unknown')}")
-print(f"Expiration Date: {whois_info.get('expiration_date', 'Unknown')}")
-print(f"Updated Date: {whois_info.get('updated_date', 'Unknown')}")
-
-# Format as text
-whois_text = f"WHOIS Information for domain: {domain}\n\n"
-for key, value in whois_info.items():
-    if value:
-        whois_text += f"{key}: {value}\n"
-
-# Print the first 200 characters
-print("\nFormatted WHOIS data (first 200 chars):")
-print(whois_text[:200] + "...")
-
-print("\nWHOIS lookup successful!")