Merge branch 'master' into conditions-then-plugins

.github/test/Dockerfile-alpine

@@ -8,26 +8,27 @@ ENV PYTHONUNBUFFERED=1
 COPY requirements.txt /requirements.txt
 
 RUN \
-  apk add --update --no-cache --virtual=build-dependencies \
+ apk add --update --no-cache --virtual=build-dependencies \
+    build-base \
     cargo \
-    g++ \
+    git \
-    gcc \
     jpeg-dev \
     libc-dev \
     libffi-dev \
-    libjpeg \
     libxslt-dev \
-    make \
     openssl-dev \
-    py3-wheel \
     python3-dev \
+    zip \
     zlib-dev && \
   apk add --update --no-cache \
+    libjpeg \
     libxslt \
-    python3 \
+    nodejs \
-    py3-pip && \
+    poppler-utils \
+    python3 && \
   echo "**** pip3 install test of changedetection.io ****" && \
-  pip3 install -U pip wheel setuptools && \
+  python3 -m venv /lsiopy  && \
-  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.21/ -r /requirements.txt && \
+  pip install -U pip wheel setuptools && \
+  pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.21/ -r /requirements.txt && \
   apk del --purge \
     build-dependencies

.github/workflows/containers.yml

@@ -103,6 +103,19 @@ jobs:
 #          provenance: false
 
       # A new tagged release is required, which builds :tag and :latest
+      - name: Docker meta :tag
+        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
+        uses: docker/metadata-action@v5
+        id: meta
+        with:
+            images: |
+                ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
+                ghcr.io/dgtlmoon/changedetection.io
+            tags: |
+                type=semver,pattern={{version}}
+                type=semver,pattern={{major}}.{{minor}}
+                type=semver,pattern={{major}}
+
       - name: Build and push :tag
         id: docker_build_tag_release
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
@@ -111,11 +124,7 @@ jobs:
           context: ./
           file: ./Dockerfile
           push: true
-          tags: |
+          tags: ${{ steps.meta.outputs.tags }}
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
-            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
-            ghcr.io/dgtlmoon/changedetection.io:latest
           platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max

.github/workflows/pypi-release.yml

@@ -45,9 +45,12 @@ jobs:
     - name: Test that the basic pip built package runs without error
       run: |
         set -ex
-        sudo pip3 install --upgrade pip 
+        ls -alR 
-        pip3 install dist/changedetection.io*.whl
+        
+        # Find and install the first .whl file
+        find dist -type f -name "*.whl" -exec pip3 install {} \; -quit
         changedetection.io -d /tmp -p 10000 &
+        
         sleep 3
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/ >/dev/null

.github/workflows/test-stack-reusable-workflow.yml

@@ -64,14 +64,16 @@ jobs:
           echo "Running processes in docker..."
           docker ps
 
-      - name: Test built container with Pytest (generally as requests/plaintext fetching)
+      - name: Run Unit Tests
         run: |
           # Unit tests
-          echo "run test with unittest"
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
-          
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
+
+      - name: Test built container with Pytest (generally as requests/plaintext fetching)
+        run: |
           # All tests
           echo "run test with pytest"
           # The default pytest logger_level is TRACE

MANIFEST.in

@@ -2,6 +2,7 @@ recursive-include changedetectionio/api *
 recursive-include changedetectionio/apprise_plugin *
 recursive-include changedetectionio/blueprint *
 recursive-include changedetectionio/content_fetchers *
+recursive-include changedetectionio/conditions *
 recursive-include changedetectionio/model *
 recursive-include changedetectionio/processors *
 recursive-include changedetectionio/static *

README.md

@@ -120,7 +120,7 @@ Easily add the current web page to your changedetection.io tool, simply install
 
 [<img src="./docs/chrome-extension-screenshot.png" style="max-width:80%;" alt="Chrome Extension to easily add the current web-page to detect a change."  title="Chrome Extension to easily add the current web-page to detect a change."  />](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
 
-[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
+[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop) ( Or check out the [GitHub repo](https://github.com/dgtlmoon/changedetection.io-browser-extension) ) 
 
 ## Installation
 

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.48.05'
+__version__ = '0.49.4'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
@@ -24,6 +24,9 @@ from loguru import logger
 app = None
 datastore = None
 
+def get_version():
+    return __version__
+
 # Parent wrapper or OS sends us a SIGTERM/SIGINT, do everything required for a clean shutdown
 def sigshutdown_handler(_signo, _stack_frame):
     global app

changedetectionio/api/api_schema.py

@@ -112,6 +112,35 @@ def build_watch_json_schema(d):
 
     schema['properties']['time_between_check'] = build_time_between_check_json_schema()
 
+    schema['properties']['browser_steps'] = {
+        "anyOf": [
+            {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "operation": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000  # Allows null and any string up to 5000 chars (including "")
+                        },
+                        "selector": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000
+                        },
+                        "optional_value": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000
+                        }
+                    },
+                    "required": ["operation", "selector", "optional_value"],
+                    "additionalProperties": False  # No extra keys allowed
+                }
+            },
+            {"type": "null"},  # Allows null for `browser_steps`
+            {"type": "array", "maxItems": 0}  # Allows empty array []
+        ]
+    }
+
     # headers ?
     return schema
 

changedetectionio/api/api_v1.py

@@ -76,6 +76,7 @@ class Watch(Resource):
         # Return without history, get that via another API call
         # Properties are not returned as a JSON, so add the required props manually
         watch['history_n'] = watch.history_n
+        # attr .last_changed will check for the last written text snapshot on change
         watch['last_changed'] = watch.last_changed
         watch['viewed'] = watch.viewed
         return watch

changedetectionio/apprise_asset.py

@@ -1,3 +1,4 @@
+from changedetectionio import apprise_plugin
 import apprise
 
 # Create our AppriseAsset and populate it with some of our new values:

changedetectionio/apprise_plugin/__init__.py

@@ -1,6 +1,8 @@
 # include the decorator
 from apprise.decorators import notify
 from loguru import logger
+from requests.structures import CaseInsensitiveDict
+
 
 @notify(on="delete")
 @notify(on="deletes")
@@ -13,70 +15,84 @@ from loguru import logger
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     import requests
     import json
+    import re
+
     from urllib.parse import unquote_plus
-    from apprise.utils import parse_url as apprise_parse_url
+    from apprise.utils.parse import parse_url as apprise_parse_url
-    from apprise import URLBase
 
     url = kwargs['meta'].get('url')
+    schema = kwargs['meta'].get('schema').lower().strip()
 
-    if url.startswith('post'):
+    # Choose POST, GET etc from requests
-        r = requests.post
+    method =  re.sub(rf's$', '', schema)
-    elif url.startswith('get'):
+    requests_method = getattr(requests, method)
-        r = requests.get
-    elif url.startswith('put'):
-        r = requests.put
-    elif url.startswith('delete'):
-        r = requests.delete
 
-    url = url.replace('post://', 'http://')
+    params = CaseInsensitiveDict({}) # Added to requests
-    url = url.replace('posts://', 'https://')
-    url = url.replace('put://', 'http://')
-    url = url.replace('puts://', 'https://')
-    url = url.replace('get://', 'http://')
-    url = url.replace('gets://', 'https://')
-    url = url.replace('put://', 'http://')
-    url = url.replace('puts://', 'https://')
-    url = url.replace('delete://', 'http://')
-    url = url.replace('deletes://', 'https://')
-
-    headers = {}
-    params = {}
     auth = None
+    has_error = False
 
     # Convert /foobar?+some-header=hello to proper header dictionary
     results = apprise_parse_url(url)
-    if results:
-        # Add our headers that the user can potentially over-ride if they wish
-        # to to our returned result set and tidy entries by unquoting them
-        headers = {unquote_plus(x): unquote_plus(y)
-                   for x, y in results['qsd+'].items()}
 
-        # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+    # Add our headers that the user can potentially over-ride if they wish
-        # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
+    # to to our returned result set and tidy entries by unquoting them
-        # but here we are making straight requests, so we need todo convert this against apprise's logic
+    headers = CaseInsensitiveDict({unquote_plus(x): unquote_plus(y)
-        for k, v in results['qsd'].items():
+               for x, y in results['qsd+'].items()})
-            if not k.strip('+-') in results['qsd+'].keys():
-                params[unquote_plus(k)] = unquote_plus(v)
 
-        # Determine Authentication
+    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-        auth = ''
+    # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
-        if results.get('user') and results.get('password'):
+    # but here we are making straight requests, so we need todo convert this against apprise's logic
-            auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
+    for k, v in results['qsd'].items():
-        elif results.get('user'):
+        if not k.strip('+-') in results['qsd+'].keys():
-            auth = (unquote_plus(results.get('user')))
+            params[unquote_plus(k)] = unquote_plus(v)
 
-    # Try to auto-guess if it's JSON
+    # Determine Authentication
-    h = 'application/json; charset=utf-8'
+    auth = ''
+    if results.get('user') and results.get('password'):
+        auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
+    elif results.get('user'):
+        auth = (unquote_plus(results.get('user')))
+
+    # If it smells like it could be JSON and no content-type was already set, offer a default content type.
+    if body and '{' in body[:100] and not headers.get('Content-Type'):
+        json_header = 'application/json; charset=utf-8'
+        try:
+            # Try if it's JSON
+            json.loads(body)
+            headers['Content-Type'] = json_header
+        except ValueError as e:
+            logger.warning(f"Could not automatically add '{json_header}' header to the notification because the document failed to parse as JSON: {e}")
+            pass
+
+    # POSTS -> HTTPS etc
+    if schema.lower().endswith('s'):
+        url = re.sub(rf'^{schema}', 'https', results.get('url'))
+    else:
+        url = re.sub(rf'^{schema}', 'http', results.get('url'))
+
+    status_str = ''
     try:
-        json.loads(body)
+        r = requests_method(url,
-        headers['Content-Type'] = h
+          auth=auth,
-    except ValueError as e:
+          data=body.encode('utf-8') if type(body) is str else body,
-        logger.warning(f"Could not automatically add '{h}' header to the {kwargs['meta'].get('schema')}:// notification because the document failed to parse as JSON: {e}")
+          headers=headers,
-        pass
+          params=params
+        )
 
-    r(results.get('url'),
+        if not (200 <= r.status_code < 300):
-      auth=auth,
+            status_str = f"Error sending '{method.upper()}' request to {url} - Status: {r.status_code}: '{r.reason}'"
-      data=body.encode('utf-8') if type(body) is str else body,
+            logger.error(status_str)
-      headers=headers,
+            has_error = True
-      params=params
+        else:
-      )
+            logger.info(f"Sent '{method.upper()}' request to {url}")
+            has_error = False
+
+    except requests.RequestException as e:
+        status_str = f"Error sending '{method.upper()}' request to {url} - {str(e)}"
+        logger.error(status_str)
+        has_error = True
+
+    if has_error:
+        raise TypeError(status_str)
+
+    return True

changedetectionio/blueprint/browser_steps/__init__.py

@@ -22,7 +22,10 @@ from loguru import logger
 
 browsersteps_sessions = {}
 io_interface_context = None
-
+import json
+import base64
+import hashlib
+from flask import Response
 
 def construct_blueprint(datastore: ChangeDetectionStore):
     browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
@@ -85,7 +88,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui(
             playwright_browser=browsersteps_start_session['browser'],
             proxy=proxy,
-            start_url=datastore.data['watching'][watch_uuid].get('url'),
+            start_url=datastore.data['watching'][watch_uuid].link,
             headers=datastore.data['watching'][watch_uuid].get('headers')
         )
 
@@ -160,14 +163,13 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         if not browsersteps_sessions.get(browsersteps_session_id):
             return make_response('No session exists under that ID', 500)
 
-
+        is_last_step = False
         # Actions - step/apply/etc, do the thing and return state
         if request.method == 'POST':
             # @todo - should always be an existing session
             step_operation = request.form.get('operation')
             step_selector = request.form.get('selector')
             step_optional_value = request.form.get('optional_value')
-            step_n = int(request.form.get('step_n'))
             is_last_step = strtobool(request.form.get('is_last_step'))
 
             # @todo try.. accept.. nice errors not popups..
@@ -182,16 +184,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 # Try to find something of value to give back to the user
                 return make_response(str(e).splitlines()[0], 401)
 
-            # Get visual selector ready/update its data (also use the current filter info from the page?)
-            # When the last 'apply' button was pressed
-            # @todo this adds overhead because the xpath selection is happening twice
-            u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
-            if is_last_step and u:
-                (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data()
-                watch = datastore.data['watching'].get(uuid)
-                if watch:
-                    watch.save_screenshot(screenshot=screenshot)
-                    watch.save_xpath_data(data=xpath_data)
 
 #        if not this_session.page:
 #            cleanup_playwright_session()
@@ -199,31 +191,35 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         # Screenshots and other info only needed on requesting a step (POST)
         try:
-            state = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
+            (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
+            if is_last_step:
+                watch = datastore.data['watching'].get(uuid)
+                u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
+                if watch and u:
+                    watch.save_screenshot(screenshot=screenshot)
+                    watch.save_xpath_data(data=xpath_data)
+
         except playwright._impl._api_types.Error as e:
             return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
+        except Exception as e:
+            return make_response("Error fetching screenshot and element data - " + str(e), 401)
 
-        # Use send_file() which is way faster than read/write loop on bytes
+        # SEND THIS BACK TO THE BROWSER
-        import json
-        from tempfile import mkstemp
-        from flask import send_file
-        tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
 
-        output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
+        output = {
-            base64.b64encode(state[0]).decode('ascii')),
+            "screenshot": f"data:image/jpeg;base64,{base64.b64encode(screenshot).decode('ascii')}",
-            'xpath_data': state[1],
+            "xpath_data": xpath_data,
-            'session_age_start': browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
+            "session_age_start": browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
-            'browser_time_remaining': round(remaining)
+            "browser_time_remaining": round(remaining)
-        })
+        }
+        json_data = json.dumps(output)
 
-        with os.fdopen(tmp_fd, 'w') as f:
+        # Generate an ETag (hash of the response body)
-            f.write(output)
+        etag_hash = hashlib.md5(json_data.encode('utf-8')).hexdigest()
 
-        response = make_response(send_file(path_or_file=tmp_file,
+        # Create the response with ETag
-                                           mimetype='application/json; charset=UTF-8',
+        response = Response(json_data, mimetype="application/json; charset=UTF-8")
-                                           etag=True))
+        response.set_etag(etag_hash)
-        # No longer needed
-        os.unlink(tmp_file)
 
         return response
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -1,14 +1,15 @@
-#!/usr/bin/env python3
-
 import os
 import time
 import re
 from random import randint
 from loguru import logger
 
+from changedetectionio.content_fetchers.helpers import capture_stitched_together_full_page, SCREENSHOT_SIZE_STITCH_THRESHOLD
 from changedetectionio.content_fetchers.base import manage_user_agent
 from changedetectionio.safe_jinja import render as jinja_render
 
+
+
 # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
 # 0- off, 1- on
 browser_step_ui_config = {'Choose one': '0 0',
@@ -31,6 +32,7 @@ browser_step_ui_config = {'Choose one': '0 0',
 #                          'Extract text and use as filter': '1 0',
                           'Goto site': '0 0',
                           'Goto URL': '0 1',
+                          'Make all child elements visible': '1 0',
                           'Press Enter': '0 0',
                           'Select by label': '1 1',
                           'Scroll down': '0 0',
@@ -38,6 +40,7 @@ browser_step_ui_config = {'Choose one': '0 0',
                           'Wait for seconds': '0 1',
                           'Wait for text': '0 1',
                           'Wait for text in element': '1 1',
+                          'Remove elements': '1 0',
                           #                          'Press Page Down': '0 0',
                           #                          'Press Page Up': '0 0',
                           # weird bug, come back to it later
@@ -52,6 +55,8 @@ class steppable_browser_interface():
     page = None
     start_url = None
 
+    action_timeout = 10 * 1000
+
     def __init__(self, start_url):
         self.start_url = start_url
 
@@ -102,7 +107,7 @@ class steppable_browser_interface():
             return
         elem = self.page.get_by_text(value)
         if elem.count():
-            elem.first.click(delay=randint(200, 500), timeout=3000)
+            elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
 
     def action_click_element_containing_text_if_exists(self, selector=None, value=''):
         logger.debug("Clicking element containing text if exists")
@@ -111,7 +116,7 @@ class steppable_browser_interface():
         elem = self.page.get_by_text(value)
         logger.debug(f"Clicking element containing text - {elem.count()} elements found")
         if elem.count():
-            elem.first.click(delay=randint(200, 500), timeout=3000)
+            elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
         else:
             return
 
@@ -119,7 +124,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
 
-        self.page.fill(selector, value, timeout=10 * 1000)
+        self.page.fill(selector, value, timeout=self.action_timeout)
 
     def action_execute_js(self, selector, value):
         response = self.page.evaluate(value)
@@ -130,7 +135,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
 
-        self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))
+        self.page.click(selector=selector, timeout=self.action_timeout + 20 * 1000, delay=randint(200, 500))
 
     def action_click_element_if_exists(self, selector, value):
         import playwright._impl._errors as _api_types
@@ -138,7 +143,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
         try:
-            self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
+            self.page.click(selector, timeout=self.action_timeout, delay=randint(200, 500))
         except _api_types.TimeoutError as e:
             return
         except _api_types.Error as e:
@@ -185,11 +190,29 @@ class steppable_browser_interface():
         self.page.keyboard.press("PageDown", delay=randint(200, 500))
 
     def action_check_checkbox(self, selector, value):
-        self.page.locator(selector).check(timeout=1000)
+        self.page.locator(selector).check(timeout=self.action_timeout)
 
     def action_uncheck_checkbox(self, selector, value):
-        self.page.locator(selector, timeout=1000).uncheck(timeout=1000)
+        self.page.locator(selector).uncheck(timeout=self.action_timeout)
 
+    def action_remove_elements(self, selector, value):
+        """Removes all elements matching the given selector from the DOM."""
+        self.page.locator(selector).evaluate_all("els => els.forEach(el => el.remove())")
+
+    def action_make_all_child_elements_visible(self, selector, value):
+        """Recursively makes all child elements inside the given selector fully visible."""
+        self.page.locator(selector).locator("*").evaluate_all("""
+            els => els.forEach(el => {
+                el.style.display = 'block';   // Forces it to be displayed
+                el.style.visibility = 'visible';   // Ensures it's not hidden
+                el.style.opacity = '1';   // Fully opaque
+                el.style.position = 'relative';   // Avoids 'absolute' hiding
+                el.style.height = 'auto';   // Expands collapsed elements
+                el.style.width = 'auto';   // Ensures full visibility
+                el.removeAttribute('hidden');   // Removes hidden attribute
+                el.classList.remove('hidden', 'd-none');  // Removes common CSS hidden classes
+            })
+        """)
 
 # Responsible for maintaining a live 'context' with the chrome CDP
 # @todo - how long do contexts live for anyway?
@@ -257,6 +280,7 @@ class browsersteps_live_ui(steppable_browser_interface):
         logger.debug(f"Time to browser setup {time.time()-now:.2f}s")
         self.page.wait_for_timeout(1 * 1000)
 
+
     def mark_as_closed(self):
         logger.debug("Page closed, cleaning up..")
 
@@ -274,39 +298,30 @@ class browsersteps_live_ui(steppable_browser_interface):
         now = time.time()
         self.page.wait_for_timeout(1 * 1000)
 
-        # The actual screenshot
-        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
 
+        full_height = self.page.evaluate("document.documentElement.scrollHeight")
+
+        if full_height >= SCREENSHOT_SIZE_STITCH_THRESHOLD:
+            logger.warning(f"Page full Height: {full_height}px longer than {SCREENSHOT_SIZE_STITCH_THRESHOLD}px, using 'stitched screenshot method'.")
+            screenshot = capture_stitched_together_full_page(self.page)
+        else:
+            screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
+
+        logger.debug(f"Time to get screenshot from browser {time.time() - now:.2f}s")
+
+        now = time.time()
         self.page.evaluate("var include_filters=''")
         # Go find the interactive elements
         # @todo in the future, something smarter that can scan for elements with .click/focus etc event handlers?
         elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
         xpath_element_js = xpath_element_js.replace('%ELEMENTS%', elements)
+
         xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
         # So the JS will find the smallest one first
         xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
-        logger.debug(f"Time to complete get_current_state of browser {time.time()-now:.2f}s")
+        logger.debug(f"Time to scrape xpath element data in browser {time.time()-now:.2f}s")
-        # except
+
         # playwright._impl._api_types.Error: Browser closed.
         # @todo show some countdown timer?
         return (screenshot, xpath_data)
 
-    def request_visualselector_data(self):
-        """
-        Does the same that the playwright operation in content_fetcher does
-        This is used to just bump the VisualSelector data so it' ready to go if they click on the tab
-        @todo refactor and remove duplicate code, add include_filters
-        :param xpath_data:
-        :param screenshot:
-        :param current_include_filters:
-        :return:
-        """
-        import importlib.resources
-        self.page.evaluate("var include_filters=''")
-        xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
-        from changedetectionio.content_fetchers import visualselector_xpath_selectors
-        xpath_element_js = xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
-        xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
-        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
-
-        return (screenshot, xpath_data)

changedetectionio/conditions/__init__.py

@@ -0,0 +1,135 @@
+from flask import Blueprint
+
+from json_logic.builtins import BUILTINS
+
+from .exceptions import EmptyConditionRuleRowNotUsable
+from .pluggy_interface import plugin_manager  # Import the pluggy plugin manager
+from . import default_plugin
+
+# List of all supported JSON Logic operators
+operator_choices = [
+    (None, "Choose one"),
+    (">", "Greater Than"),
+    ("<", "Less Than"),
+    (">=", "Greater Than or Equal To"),
+    ("<=", "Less Than or Equal To"),
+    ("==", "Equals"),
+    ("!=", "Not Equals"),
+    ("in", "Contains"),
+    ("!in", "Does Not Contain"),
+]
+
+# Fields available in the rules
+field_choices = [
+    (None, "Choose one"),
+]
+
+# The data we will feed the JSON Rules to see if it passes the test/conditions or not
+EXECUTE_DATA = {}
+
+
+# Define the extended operations dictionary
+CUSTOM_OPERATIONS = {
+    **BUILTINS,  # Include all standard operators
+}
+
+def filter_complete_rules(ruleset):
+    rules = [
+        rule for rule in ruleset
+        if all(value not in ("", False, "None", None) for value in [rule["operator"], rule["field"], rule["value"]])
+    ]
+    return rules
+
+def convert_to_jsonlogic(logic_operator: str, rule_dict: list):
+    """
+    Convert a structured rule dict into a JSON Logic rule.
+
+    :param rule_dict: Dictionary containing conditions.
+    :return: JSON Logic rule as a dictionary.
+    """
+
+
+    json_logic_conditions = []
+
+    for condition in rule_dict:
+        operator = condition["operator"]
+        field = condition["field"]
+        value = condition["value"]
+
+        if not operator or operator == 'None' or not value or not field:
+            raise EmptyConditionRuleRowNotUsable()
+
+        # Convert value to int/float if possible
+        try:
+            if isinstance(value, str) and "." in value and str != "None":
+                value = float(value)
+            else:
+                value = int(value)
+        except (ValueError, TypeError):
+            pass  # Keep as a string if conversion fails
+
+        # Handle different JSON Logic operators properly
+        if operator == "in":
+            json_logic_conditions.append({"in": [value, {"var": field}]})  # value first
+        elif operator in ("!", "!!", "-"):
+            json_logic_conditions.append({operator: [{"var": field}]})  # Unary operators
+        elif operator in ("min", "max", "cat"):
+            json_logic_conditions.append({operator: value})  # Multi-argument operators
+        else:
+            json_logic_conditions.append({operator: [{"var": field}, value]})  # Standard binary operators
+
+    return {logic_operator: json_logic_conditions} if len(json_logic_conditions) > 1 else json_logic_conditions[0]
+
+
+def execute_ruleset_against_all_plugins(current_watch_uuid: str, application_datastruct, ephemeral_data={} ):
+    """
+    Build our data and options by calling our plugins then pass it to jsonlogic and see if the conditions pass
+
+    :param ruleset: JSON Logic rule dictionary.
+    :param extracted_data: Dictionary containing the facts.   <-- maybe the app struct+uuid
+    :return: Dictionary of plugin results.
+    """
+    from json_logic import jsonLogic
+
+    EXECUTE_DATA = {}
+    result = True
+    
+    ruleset_settings = application_datastruct['watching'].get(current_watch_uuid)
+
+    if ruleset_settings.get("conditions"):
+        logic_operator = "and" if ruleset_settings.get("conditions_match_logic", "ALL") == "ALL" else "or"
+        complete_rules = filter_complete_rules(ruleset_settings['conditions'])
+        if complete_rules:
+            # Give all plugins a chance to update the data dict again (that we will test the conditions against)
+            for plugin in plugin_manager.get_plugins():
+                new_execute_data = plugin.add_data(current_watch_uuid=current_watch_uuid,
+                                                   application_datastruct=application_datastruct,
+                                                   ephemeral_data=ephemeral_data)
+
+                if new_execute_data and isinstance(new_execute_data, dict):
+                    EXECUTE_DATA.update(new_execute_data)
+
+            # Create the ruleset
+            ruleset = convert_to_jsonlogic(logic_operator=logic_operator, rule_dict=complete_rules)
+            
+            # Pass the custom operations dictionary to jsonLogic
+            if not jsonLogic(logic=ruleset, data=EXECUTE_DATA, operations=CUSTOM_OPERATIONS):
+                result = False
+
+    return result
+
+
+# Load plugins dynamically
+for plugin in plugin_manager.get_plugins():
+    new_ops = plugin.register_operators()
+    if isinstance(new_ops, dict):
+        CUSTOM_OPERATIONS.update(new_ops)
+
+    new_operator_choices = plugin.register_operator_choices()
+    if isinstance(new_operator_choices, list):
+        operator_choices.extend(new_operator_choices)
+
+    new_field_choices = plugin.register_field_choices()
+    if isinstance(new_field_choices, list):
+        field_choices.extend(new_field_choices)
+

changedetectionio/conditions/blueprint.py

@@ -0,0 +1,78 @@
+# Flask Blueprint Definition
+import json
+
+from flask import Blueprint
+
+from changedetectionio.conditions import execute_ruleset_against_all_plugins
+
+
+def construct_blueprint(datastore):
+    from changedetectionio.flask_app import login_optionally_required
+
+    conditions_blueprint = Blueprint('conditions', __name__, template_folder="templates")
+
+    @conditions_blueprint.route("/<string:watch_uuid>/verify-condition-single-rule", methods=['POST'])
+    @login_optionally_required
+    def verify_condition_single_rule(watch_uuid):
+        """Verify a single condition rule against the current snapshot"""
+        from changedetectionio.processors.text_json_diff import prepare_filter_prevew
+        from flask import request, jsonify
+        from copy import deepcopy
+
+        ephemeral_data = {}
+
+        # Get the watch data
+        watch = datastore.data['watching'].get(watch_uuid)
+        if not watch:
+            return jsonify({'status': 'error', 'message': 'Watch not found'}), 404
+
+        # First use prepare_filter_prevew to process the form data
+        # This will return text_after_filter which is after all current form settings are applied
+        # Create ephemeral data with the text from the current snapshot
+
+        try:
+            # Call prepare_filter_prevew to get a processed version of the content with current form settings
+            # We'll ignore the returned response and just use the datastore which is modified by the function
+
+            # this should apply all filters etc so then we can run the CONDITIONS against the final output text
+            result = prepare_filter_prevew(datastore=datastore,
+                                           form_data=request.form,
+                                           watch_uuid=watch_uuid)
+
+            ephemeral_data['text'] = result.get('after_filter', '')
+            # Create a temporary watch data structure with this single rule
+            tmp_watch_data = deepcopy(datastore.data['watching'].get(watch_uuid))
+
+            # Override the conditions in the temporary watch
+            rule_json = request.args.get("rule")
+            rule = json.loads(rule_json) if rule_json else None
+            tmp_watch_data['conditions'] = [rule]
+            tmp_watch_data['conditions_match_logic'] = "ALL"  # Single rule, so use ALL
+
+            # Create a temporary application data structure for the rule check
+            temp_app_data = {
+                'watching': {
+                    watch_uuid: tmp_watch_data
+                }
+            }
+
+            # Execute the rule against the current snapshot with form data
+            result = execute_ruleset_against_all_plugins(
+                current_watch_uuid=watch_uuid,
+                application_datastruct=temp_app_data,
+                ephemeral_data=ephemeral_data
+            )
+
+            return jsonify({
+                'status': 'success',
+                'result': result,
+                'message': 'Condition passes' if result else 'Condition does not pass'
+            })
+
+        except Exception as e:
+            return jsonify({
+                'status': 'error',
+                'message': f'Error verifying condition: {str(e)}'
+            }), 500
+
+    return conditions_blueprint

changedetectionio/conditions/default_plugin.py

@@ -0,0 +1,78 @@
+import re
+
+import pluggy
+from price_parser import Price
+from loguru import logger
+
+hookimpl = pluggy.HookimplMarker("changedetectionio_conditions")
+
+
+@hookimpl
+def register_operators():
+    def starts_with(_, text, prefix):
+        return text.lower().strip().startswith(str(prefix).strip().lower())
+
+    def ends_with(_, text, suffix):
+        return text.lower().strip().endswith(str(suffix).strip().lower())
+
+    def length_min(_, text, strlen):
+        return len(text) >= int(strlen)
+
+    def length_max(_, text, strlen):
+        return len(text) <= int(strlen)
+
+    # ✅ Custom function for case-insensitive regex matching
+    def contains_regex(_, text, pattern):
+        """Returns True if `text` contains `pattern` (case-insensitive regex match)."""
+        return bool(re.search(pattern, str(text), re.IGNORECASE))
+
+    # ✅ Custom function for NOT matching case-insensitive regex
+    def not_contains_regex(_, text, pattern):
+        """Returns True if `text` does NOT contain `pattern` (case-insensitive regex match)."""
+        return not bool(re.search(pattern, str(text), re.IGNORECASE))
+
+    return {
+        "!contains_regex": not_contains_regex,
+        "contains_regex": contains_regex,
+        "ends_with": ends_with,
+        "length_max": length_max,
+        "length_min": length_min,
+        "starts_with": starts_with,
+    }
+
+@hookimpl
+def register_operator_choices():
+    return [
+        ("starts_with", "Text Starts With"),
+        ("ends_with", "Text Ends With"),
+        ("length_min", "Length minimum"),
+        ("length_max", "Length maximum"),
+        ("contains_regex", "Text Matches Regex"),
+        ("!contains_regex", "Text Does NOT Match Regex"),
+    ]
+
+@hookimpl
+def register_field_choices():
+    return [
+        ("extracted_number", "Extracted number after 'Filters & Triggers'"),
+#        ("meta_description", "Meta Description"),
+#        ("meta_keywords", "Meta Keywords"),
+        ("page_filtered_text", "Page text after 'Filters & Triggers'"),
+        #("page_title", "Page <title>"), # actual page title <title>
+    ]
+
+@hookimpl
+def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
+
+    res = {}
+    if 'text' in ephemeral_data:
+        res['page_filtered_text'] = ephemeral_data['text']
+
+        # Better to not wrap this in try/except so that the UI can see any errors
+        price = Price.fromstring(ephemeral_data.get('text'))
+        if price and price.amount != None:
+            # This is slightly misleading, it's extracting a PRICE not a Number..
+            res['extracted_number'] = float(price.amount)
+            logger.debug(f"Extracted number result: '{price}' - returning float({res['extracted_number']})")
+
+    return res

changedetectionio/conditions/exceptions.py

@@ -0,0 +1,6 @@
+class EmptyConditionRuleRowNotUsable(Exception):
+    def __init__(self):
+        super().__init__("One of the 'conditions' rulesets is incomplete, cannot run.")
+
+    def __str__(self):
+        return self.args[0]

changedetectionio/conditions/form.py

@@ -0,0 +1,44 @@
+# Condition Rule Form (for each rule row)
+from wtforms import Form, SelectField, StringField, validators
+from wtforms import validators
+
+class ConditionFormRow(Form):
+
+    # ✅ Ensure Plugins Are Loaded BEFORE Importing Choices
+    from changedetectionio.conditions import plugin_manager
+    from changedetectionio.conditions import operator_choices, field_choices
+    field = SelectField(
+        "Field",
+        choices=field_choices,
+        validators=[validators.Optional()]
+    )
+
+    operator = SelectField(
+        "Operator",
+        choices=operator_choices,
+        validators=[validators.Optional()]
+    )
+
+    value = StringField("Value", validators=[validators.Optional()])
+
+    def validate(self, extra_validators=None):
+        # First, run the default validators
+        if not super().validate(extra_validators):
+            return False
+
+        # Custom validation logic
+        # If any of the operator/field/value is set, then they must be all set
+        if any(value not in ("", False, "None", None) for value in [self.operator.data, self.field.data, self.value.data]):
+            if not self.operator.data or self.operator.data == 'None':
+                self.operator.errors.append("Operator is required.")
+                return False
+
+            if not self.field.data or self.field.data == 'None':
+                self.field.errors.append("Field is required.")
+                return False
+
+            if not self.value.data:
+                self.value.errors.append("Value is required.")
+                return False
+
+        return True  # Only return True if all conditions pass

changedetectionio/conditions/pluggy_interface.py

@@ -0,0 +1,44 @@
+import pluggy
+from . import default_plugin  # Import the default plugin
+
+# ✅ Ensure that the namespace in HookspecMarker matches PluginManager
+PLUGIN_NAMESPACE = "changedetectionio_conditions"
+
+hookspec = pluggy.HookspecMarker(PLUGIN_NAMESPACE)
+hookimpl = pluggy.HookimplMarker(PLUGIN_NAMESPACE)
+
+
+class ConditionsSpec:
+    """Hook specifications for extending JSON Logic conditions."""
+
+    @hookspec
+    def register_operators():
+        """Return a dictionary of new JSON Logic operators."""
+        pass
+
+    @hookspec
+    def register_operator_choices():
+        """Return a list of new operator choices."""
+        pass
+
+    @hookspec
+    def register_field_choices():
+        """Return a list of new field choices."""
+        pass
+
+    @hookspec
+    def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
+        """Add to the datadict"""
+        pass
+
+# ✅ Set up Pluggy Plugin Manager
+plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE)
+
+# ✅ Register hookspecs (Ensures they are detected)
+plugin_manager.add_hookspecs(ConditionsSpec)
+
+# ✅ Register built-in plugins manually
+plugin_manager.register(default_plugin, "default_plugin")
+
+# ✅ Discover installed plugins from external packages (if any)
+plugin_manager.load_setuptools_entrypoints(PLUGIN_NAMESPACE)

changedetectionio/content_fetchers/helpers.py

@@ -0,0 +1,104 @@
+
+# Pages with a vertical height longer than this will use the 'stitch together' method.
+
+# - Many GPUs have a max texture size of 16384x16384px (or lower on older devices).
+# - If a page is taller than ~8000–10000px, it risks exceeding GPU memory limits.
+# - This is especially important on headless Chromium, where Playwright may fail to allocate a massive full-page buffer.
+
+
+# The size at which we will switch to stitching method
+SCREENSHOT_SIZE_STITCH_THRESHOLD=8000
+
+from loguru import logger
+
+def capture_stitched_together_full_page(page):
+    import io
+    import os
+    import time
+    from PIL import Image, ImageDraw, ImageFont
+
+    MAX_TOTAL_HEIGHT = SCREENSHOT_SIZE_STITCH_THRESHOLD*4  # Maximum total height for the final image (When in stitch mode)
+    MAX_CHUNK_HEIGHT = 4000  # Height per screenshot chunk
+    WARNING_TEXT_HEIGHT = 20  # Height of the warning text overlay
+
+    # Save the original viewport size
+    original_viewport = page.viewport_size
+    now = time.time()
+
+    try:
+        viewport = page.viewport_size
+        page_height = page.evaluate("document.documentElement.scrollHeight")
+
+        # Limit the total capture height
+        capture_height = min(page_height, MAX_TOTAL_HEIGHT)
+
+        images = []
+        total_captured_height = 0
+
+        for offset in range(0, capture_height, MAX_CHUNK_HEIGHT):
+            # Ensure we do not exceed the total height limit
+            chunk_height = min(MAX_CHUNK_HEIGHT, MAX_TOTAL_HEIGHT - total_captured_height)
+
+            # Adjust viewport size for this chunk
+            page.set_viewport_size({"width": viewport["width"], "height": chunk_height})
+
+            # Scroll to the correct position
+            page.evaluate(f"window.scrollTo(0, {offset})")
+
+            # Capture screenshot chunk
+            screenshot_bytes = page.screenshot(type='jpeg', quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
+            images.append(Image.open(io.BytesIO(screenshot_bytes)))
+
+            total_captured_height += chunk_height
+
+            # Stop if we reached the maximum total height
+            if total_captured_height >= MAX_TOTAL_HEIGHT:
+                break
+
+        # Create the final stitched image
+        stitched_image = Image.new('RGB', (viewport["width"], total_captured_height))
+        y_offset = 0
+
+        # Stitch the screenshot chunks together
+        for img in images:
+            stitched_image.paste(img, (0, y_offset))
+            y_offset += img.height
+
+        logger.debug(f"Screenshot stitched together in {time.time()-now:.2f}s")
+
+        # Overlay warning text if the screenshot was trimmed
+        if page_height > MAX_TOTAL_HEIGHT:
+            draw = ImageDraw.Draw(stitched_image)
+            warning_text = f"WARNING: Screenshot was {page_height}px but trimmed to {MAX_TOTAL_HEIGHT}px because it was too long"
+
+            # Load font (default system font if Arial is unavailable)
+            try:
+                font = ImageFont.truetype("arial.ttf", WARNING_TEXT_HEIGHT)  # Arial (Windows/Mac)
+            except IOError:
+                font = ImageFont.load_default()  # Default font if Arial not found
+
+            # Get text bounding box (correct method for newer Pillow versions)
+            text_bbox = draw.textbbox((0, 0), warning_text, font=font)
+            text_width = text_bbox[2] - text_bbox[0]  # Calculate text width
+            text_height = text_bbox[3] - text_bbox[1]  # Calculate text height
+
+            # Define background rectangle (top of the image)
+            draw.rectangle([(0, 0), (viewport["width"], WARNING_TEXT_HEIGHT)], fill="white")
+
+            # Center text horizontally within the warning area
+            text_x = (viewport["width"] - text_width) // 2
+            text_y = (WARNING_TEXT_HEIGHT - text_height) // 2
+
+            # Draw the warning text in red
+            draw.text((text_x, text_y), warning_text, fill="red", font=font)
+
+        # Save or return the final image
+        output = io.BytesIO()
+        stitched_image.save(output, format="JPEG", quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
+        screenshot = output.getvalue()
+
+    finally:
+        # Restore the original viewport size
+        page.set_viewport_size(original_viewport)
+
+    return screenshot

changedetectionio/content_fetchers/playwright.py

@@ -4,6 +4,7 @@ from urllib.parse import urlparse
 
 from loguru import logger
 
+from changedetectionio.content_fetchers.helpers import capture_stitched_together_full_page, SCREENSHOT_SIZE_STITCH_THRESHOLD
 from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable
 
@@ -89,6 +90,7 @@ class fetcher(Fetcher):
         from playwright.sync_api import sync_playwright
         import playwright._impl._errors
         from changedetectionio.content_fetchers import visualselector_xpath_selectors
+        import time
         self.delete_browser_steps_screenshots()
         response = None
 
@@ -179,6 +181,7 @@ class fetcher(Fetcher):
 
             self.page.wait_for_timeout(extra_wait * 1000)
 
+            now = time.time()
             # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
             if current_include_filters is not None:
                 self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters)))
@@ -190,6 +193,8 @@ class fetcher(Fetcher):
             self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
 
             self.content = self.page.content()
+            logger.debug(f"Time to scrape xpath element data in browser {time.time() - now:.2f}s")
+
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
             # JPEG is better here because the screenshots can be very very large
@@ -199,10 +204,15 @@ class fetcher(Fetcher):
             # acceptable screenshot quality here
             try:
                 # The actual screenshot - this always base64 and needs decoding! horrible! huge CPU usage
-                self.screenshot = self.page.screenshot(type='jpeg',
+                full_height = self.page.evaluate("document.documentElement.scrollHeight")
-                                                       full_page=True,
+
-                                                       quality=int(os.getenv("SCREENSHOT_QUALITY", 72)),
+                if full_height >= SCREENSHOT_SIZE_STITCH_THRESHOLD:
-                                                       )
+                    logger.warning(
+                        f"Page full Height: {full_height}px longer than {SCREENSHOT_SIZE_STITCH_THRESHOLD}px, using 'stitched screenshot method'.")
+                    self.screenshot = capture_stitched_together_full_page(self.page)
+                else:
+                    self.screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
+
             except Exception as e:
                 # It's likely the screenshot was too long/big and something crashed
                 raise ScreenshotUnavailable(url=url, status_code=self.status_code)

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -29,8 +29,11 @@ function isItemInStock() {
         'currently unavailable',
         'dieser artikel ist bald wieder verfügbar',
         'dostępne wkrótce',
+        'en rupture',
         'en rupture de stock',
+        'épuisé',
         'esgotado',
+        'indisponible',
         'indisponível',
         'isn\'t in stock right now',
         'isnt in stock right now',
@@ -52,6 +55,8 @@ function isItemInStock() {
         'niet leverbaar',
         'niet op voorraad',
         'no disponible',
+        'non disponibile',
+        'non disponible',
         'no longer in stock',
         'no tickets available',
         'not available',
@@ -64,8 +69,10 @@ function isItemInStock() {
         'não estamos a aceitar encomendas',
         'out of stock',
         'out-of-stock',
+        'plus disponible',
         'prodotto esaurito',
         'produkt niedostępny',
+        'rupture',
         'sold out',
         'sold-out',
         'stokta yok',

changedetectionio/content_fetchers/res/xpath_element_scraper.js

@@ -41,7 +41,7 @@ const findUpTag = (el) => {
 
     //  Strategy 1: If it's an input, with name, and there's only one, prefer that
     if (el.name !== undefined && el.name.length) {
-        var proposed = el.tagName + "[name=" + el.name + "]";
+        var proposed = el.tagName + "[name=\"" + CSS.escape(el.name) + "\"]";
         var proposed_element = window.document.querySelectorAll(proposed);
         if (proposed_element.length) {
             if (proposed_element.length === 1) {
@@ -102,13 +102,15 @@ function collectVisibleElements(parent, visibleElements) {
     const children = parent.children;
     for (let i = 0; i < children.length; i++) {
         const child = children[i];
+        const computedStyle = window.getComputedStyle(child);
+
         if (
             child.nodeType === Node.ELEMENT_NODE &&
-            window.getComputedStyle(child).display !== 'none' &&
+            computedStyle.display !== 'none' &&
-            window.getComputedStyle(child).visibility !== 'hidden' &&
+            computedStyle.visibility !== 'hidden' &&
             child.offsetWidth >= 0 &&
             child.offsetHeight >= 0 &&
-            window.getComputedStyle(child).contentVisibility !== 'hidden'
+            computedStyle.contentVisibility !== 'hidden'
         ) {
             // If the child is an element and is visible, recursively collect visible elements
             collectVisibleElements(child, visibleElements);
@@ -173,6 +175,7 @@ visibleElementsArray.forEach(function (element) {
 
     // Try to identify any possible currency amounts "Sale: 4000" or "Sale now 3000 Kc", can help with the training.
     const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6)) ) &&  /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|,–)/.test(text) ;
+    const computedStyle = window.getComputedStyle(element);
 
     size_pos.push({
         xpath: xpath_result,
@@ -184,10 +187,10 @@ visibleElementsArray.forEach(function (element) {
         tagName: (element.tagName) ? element.tagName.toLowerCase() : '',
         // tagtype used by Browser Steps
         tagtype: (element.tagName.toLowerCase() === 'input' && element.type) ? element.type.toLowerCase() : '',
-        isClickable: window.getComputedStyle(element).cursor === "pointer",
+        isClickable: computedStyle.cursor === "pointer",
         // Used by the keras trainer
-        fontSize: window.getComputedStyle(element).getPropertyValue('font-size'),
+        fontSize: computedStyle.getPropertyValue('font-size'),
-        fontWeight: window.getComputedStyle(element).getPropertyValue('font-weight'),
+        fontWeight: computedStyle.getPropertyValue('font-weight'),
         hasDigitCurrency: hasDigitCurrency,
         label: label,
     });

changedetectionio/flask_app.py

@@ -598,17 +598,31 @@ def changedetection_app(config=None, datastore_o=None):
 
             if 'notification_title' in request.form and request.form['notification_title'].strip():
                 n_object['notification_title'] = request.form.get('notification_title', '').strip()
+            elif datastore.data['settings']['application'].get('notification_title'):
+                n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
+            else:
+                n_object['notification_title'] = "Test title"
 
             if 'notification_body' in request.form and request.form['notification_body'].strip():
                 n_object['notification_body'] = request.form.get('notification_body', '').strip()
+            elif datastore.data['settings']['application'].get('notification_body'):
+                n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
+            else:
+                n_object['notification_body'] = "Test body"
 
+            n_object['as_async'] = False
             n_object.update(watch.extra_notification_token_values())
+            from .notification import process_notification
+            sent_obj = process_notification(n_object, datastore)
 
-            from . import update_worker
-            new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
-            new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch)
         except Exception as e:
-            return make_response(f"Error: str(e)", 400)
+            e_str = str(e)
+            # Remove this text which is not important and floods the container
+            e_str = e_str.replace(
+                "DEBUG - <class 'apprise.decorators.base.CustomNotifyPlugin.instantiate_plugin.<locals>.CustomNotifyPluginWrapper'>",
+                '')
+
+            return make_response(e_str, 400)
 
         return 'OK - Sent test notifications'
 
@@ -698,23 +712,63 @@ def changedetection_app(config=None, datastore_o=None):
         # Does it use some custom form? does one exist?
         processor_name = datastore.data['watching'][uuid].get('processor', '')
         processor_classes = next((tpl for tpl in find_processors() if tpl[1] == processor_name), None)
+        
+        # If it's not found in traditional processors, check if it's a pluggy plugin
         if not processor_classes:
-            flash(f"Cannot load the edit form for processor/plugin '{processor_classes[1]}', plugin missing?", 'error')
+            try:
-            return redirect(url_for('index'))
+                from changedetectionio.processors.processor_registry import get_processor_form, _get_plugin_name_map
-
+                
-        parent_module = get_parent_module(processor_classes[0])
+                # Get all available plugins for debugging
-
+                available_plugins = list(_get_plugin_name_map().keys())
-        try:
+                logger.debug(f"Available processor plugins: {available_plugins}")
-            # Get the parent of the "processor.py" go up one, get the form (kinda spaghetti but its reusing existing code)
+                
-            forms_module = importlib.import_module(f"{parent_module.__name__}.forms")
+                # Try to get the processor form
-            # Access the 'processor_settings_form' class from the 'forms' module
+                plugin_form_class = get_processor_form(processor_name)
-            form_class = getattr(forms_module, 'processor_settings_form')
+                
-        except ModuleNotFoundError as e:
+                if plugin_form_class:
-            # .forms didnt exist
+                    # Use default text_json_diff_form as parent module for plugins
-            form_class = forms.processor_text_json_diff_form
+                    from changedetectionio.processors.text_json_diff import processor as text_json_diff_processor
-        except AttributeError as e:
+                    form_class = forms.processor_text_json_diff_form
-            # .forms exists but no useful form
+                    parent_module = get_parent_module(text_json_diff_processor)
-            form_class = forms.processor_text_json_diff_form
+                    
+                    # Skip the normal form loading code path
+                    use_plugin_form = True
+                    logger.debug(f"Successfully loaded form for plugin '{processor_name}'")
+                else:
+                    # Check if the plugin is registered but doesn't have a form
+                    if processor_name in available_plugins:
+                        logger.error(f"Plugin '{processor_name}' is registered but has no form class")
+                        flash(f"Plugin '{processor_name}' is registered but has no form class", 'error')
+                    else:
+                        logger.error(f"Cannot find plugin '{processor_name}'. Available plugins: {available_plugins}")
+                        flash(f"Cannot load the edit form for processor/plugin '{processor_name}', plugin missing?", 'error')
+                    return redirect(url_for('index'))
+            except ImportError as e:
+                logger.error(f"Import error when loading plugin form: {str(e)}")
+                flash(f"Cannot load the edit form for processor/plugin '{processor_name}', plugin system not available?", 'error')
+                return redirect(url_for('index'))
+            except Exception as e:
+                logger.error(f"Unexpected error loading plugin form: {str(e)}")
+                flash(f"Error loading plugin form: {str(e)}", 'error')
+                return redirect(url_for('index'))
+        else:
+            # Traditional processor - continue with normal flow
+            parent_module = get_parent_module(processor_classes[0])
+            use_plugin_form = False
+        
+        # Only follow this path for traditional processors
+        if not use_plugin_form:
+            try:
+                # Get the parent of the "processor.py" go up one, get the form (kinda spaghetti but its reusing existing code)
+                forms_module = importlib.import_module(f"{parent_module.__name__}.forms")
+                # Access the 'processor_settings_form' class from the 'forms' module
+                form_class = getattr(forms_module, 'processor_settings_form')
+            except ModuleNotFoundError as e:
+                # .forms didnt exist
+                form_class = forms.processor_text_json_diff_form
+            except AttributeError as e:
+                # .forms exists but no useful form
+                form_class = forms.processor_text_json_diff_form
 
         form = form_class(formdata=request.form if request.method == 'POST' else None,
                           data=default,
@@ -861,14 +915,14 @@ def changedetection_app(config=None, datastore_o=None):
 
             system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
 
-            is_html_webdriver = False
+            watch_uses_webdriver = False
             if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
-                is_html_webdriver = True
+                watch_uses_webdriver = True
 
             from zoneinfo import available_timezones
 
             # Only works reliably with Playwright
-            visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver
+
             template_args = {
                 'available_processors': processors.available_processors(),
                 'available_timezones': sorted(available_timezones()),
@@ -881,14 +935,13 @@ def changedetection_app(config=None, datastore_o=None):
                 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False,
                 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
                 'has_special_tag_options': _watch_has_tag_options_set(watch=watch),
-                'is_html_webdriver': is_html_webdriver,
+                'watch_uses_webdriver': watch_uses_webdriver,
                 'jq_support': jq_support,
                 'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False),
                 'settings_application': datastore.data['settings']['application'],
                 'timezone_default_config': datastore.data['settings']['application'].get('timezone'),
                 'using_global_webdriver_wait': not default['webdriver_delay'],
                 'uuid': uuid,
-                'visualselector_enabled': visualselector_enabled,
                 'watch': watch
             }
 
@@ -1369,8 +1422,10 @@ def changedetection_app(config=None, datastore_o=None):
     @login_optionally_required
     def watch_get_preview_rendered(uuid):
         '''For when viewing the "preview" of the rendered text from inside of Edit'''
+        from flask import jsonify
         from .processors.text_json_diff import prepare_filter_prevew
-        return prepare_filter_prevew(watch_uuid=uuid, datastore=datastore)
+        result = prepare_filter_prevew(watch_uuid=uuid, form_data=request.form, datastore=datastore)
+        return jsonify(result)
 
 
     @app.route("/form/add/quickwatch", methods=['POST'])
@@ -1671,6 +1726,9 @@ def changedetection_app(config=None, datastore_o=None):
     import changedetectionio.blueprint.backups as backups
     app.register_blueprint(backups.construct_blueprint(datastore), url_prefix='/backups')
 
+    import changedetectionio.conditions.blueprint as conditions
+    app.register_blueprint(conditions.construct_blueprint(datastore), url_prefix='/conditions')
+
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
     threading.Thread(target=notification_runner).start()

changedetectionio/forms.py

@@ -3,6 +3,7 @@ import re
 from loguru import logger
 from wtforms.widgets.core import TimeInput
 
+from changedetectionio.conditions.form import ConditionFormRow
 from changedetectionio.strtobool import strtobool
 
 from wtforms import (
@@ -171,7 +172,7 @@ class validateTimeZoneName(object):
 
 class ScheduleLimitDaySubForm(Form):
     enabled = BooleanField("not set", default=True)
-    start_time = TimeStringField("Start At", default="00:00", render_kw={"placeholder": "HH:MM"}, validators=[validators.Optional()])
+    start_time = TimeStringField("Start At", default="00:00", validators=[validators.Optional()])
     duration = FormField(TimeDurationForm, label="Run duration")
 
 class ScheduleLimitForm(Form):
@@ -305,8 +306,10 @@ class ValidateAppRiseServers(object):
     def __call__(self, form, field):
         import apprise
         apobj = apprise.Apprise()
+
         # so that the custom endpoints are registered
-        from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper
+        from .apprise_asset import asset
+
         for server_url in field.data:
             url = server_url.strip()
             if url.startswith("#"):
@@ -509,6 +512,7 @@ class quickWatchForm(Form):
     edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
 
 
+
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
     from . import processors
@@ -596,6 +600,10 @@ class processor_text_json_diff_form(commonSettingsForm):
     notification_muted = BooleanField('Notifications Muted / Off', default=False)
     notification_screenshot = BooleanField('Attach screenshot to notification (where possible)', default=False)
 
+    conditions_match_logic = RadioField(u'Match', choices=[('ALL', 'Match all of the following'),('ANY', 'Match any of the following')], default='ALL')
+    conditions = FieldList(FormField(ConditionFormRow), min_entries=1)  # Add rule logic here
+
+
     def extra_tab_content(self):
         return None
 

changedetectionio/html_tools.py

@@ -1,5 +1,6 @@
-from typing import List
+from loguru import logger
 from lxml import etree
+from typing import List
 import json
 import re
 
@@ -298,8 +299,10 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 # https://github.com/dgtlmoon/changedetection.io/pull/2041#issuecomment-1848397161w
     # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded within HTML tags
     try:
-        stripped_text_from_html = _parse_json(json.loads(content), json_filter)
+        # .lstrip("\ufeff") strings ByteOrderMark from UTF8 and still lets the UTF work
-    except json.JSONDecodeError:
+        stripped_text_from_html = _parse_json(json.loads(content.lstrip("\ufeff") ), json_filter)
+    except json.JSONDecodeError as e:
+        logger.warning(str(e))
 
         # Foreach <script json></script> blob.. just return the first that matches json_filter
         # As a last resort, try to parse the whole <body>

changedetectionio/model/App.py

@@ -69,7 +69,7 @@ def parse_headers_from_text_file(filepath):
         for l in f.readlines():
             l = l.strip()
             if not l.startswith('#') and ':' in l:
-                (k, v) = l.split(':')
+                (k, v) = l.split(':', 1)  # Split only on the first colon
                 headers[k.strip()] = v.strip()
 
     return headers

changedetectionio/model/Watch.py

@@ -67,7 +67,6 @@ class model(watch_base):
 
     @property
     def link(self):
-
         url = self.get('url', '')
         if not is_safe_url(url):
             return 'DISABLED'
@@ -93,6 +92,19 @@ class model(watch_base):
         # Also double check it after any Jinja2 formatting just incase
         if not is_safe_url(ready_url):
             return 'DISABLED'
+            
+        # Check if a processor wants to customize the display link
+        processor_name = self.get('processor')
+        if processor_name:
+            try:
+                # Import here to avoid circular imports
+                from changedetectionio.processors.processor_registry import get_display_link
+                custom_link = get_display_link(url=ready_url, processor_name=processor_name)
+                if custom_link:
+                    return custom_link
+            except Exception as e:
+                logger.error(f"Error getting custom display link for processor {processor_name}: {str(e)}")
+                
         return ready_url
 
     def clear_watch(self):
@@ -352,7 +364,7 @@ class model(watch_base):
     # Iterate over all history texts and see if something new exists
     # Always applying .strip() to start/end but optionally replace any other whitespace
     def lines_contain_something_unique_compared_to_history(self, lines: list, ignore_whitespace=False):
-        local_lines = []
+        local_lines = set([])
         if lines:
             if ignore_whitespace:
                 if isinstance(lines[0], str): # Can be either str or bytes depending on what was on the disk
@@ -527,7 +539,7 @@ class model(watch_base):
     def save_error_text(self, contents):
         self.ensure_data_dir_exists()
         target_path = os.path.join(self.watch_data_dir, "last-error.txt")
-        with open(target_path, 'w') as f:
+        with open(target_path, 'w', encoding='utf-8') as f:
             f.write(contents)
 
     def save_xpath_data(self, data, as_error=False):

changedetectionio/notification.py

@@ -67,6 +67,10 @@ def process_notification(n_object, datastore):
 
     sent_objs = []
     from .apprise_asset import asset
+
+    if 'as_async' in n_object:
+        asset.async_mode = n_object.get('as_async')
+
     apobj = apprise.Apprise(debug=True, asset=asset)
 
     if not n_object.get('notification_urls'):
@@ -157,8 +161,6 @@ def process_notification(n_object, datastore):
             attach=n_object.get('screenshot', None)
         )
 
-        # Give apprise time to register an error
-        time.sleep(3)
 
         # Returns empty string if nothing found, multi-line string otherwise
         log_value = logs.getvalue()

changedetectionio/processors/__init__.py

@@ -3,6 +3,7 @@ from changedetectionio.content_fetchers.base import Fetcher
 from changedetectionio.strtobool import strtobool
 from copy import deepcopy
 from loguru import logger
+
 import hashlib
 import importlib
 import inspect
@@ -10,6 +11,10 @@ import os
 import pkgutil
 import re
 
+# Import the plugin manager
+from .pluggy_interface import plugin_manager
+
+
 class difference_detection_processor():
 
     browser_steps = None
@@ -26,9 +31,95 @@ class difference_detection_processor():
         self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
         # Generic fetcher that should be extended (requests, playwright etc)
         self.fetcher = Fetcher()
+        
+    def _get_proxy_for_watch(self, preferred_proxy_id=None):
+        """Get proxy configuration based on watch settings and preferred proxy ID
+        
+        Args:
+            preferred_proxy_id: Optional explicit proxy ID to use
+            
+        Returns:
+            dict: Proxy configuration or None if no proxy should be used
+            str: Proxy URL or None if no proxy should be used
+        """
+        # Default to no proxy config
+        proxy_config = None
+        proxy_url = None
+        
+        # Check if datastore is available and has get_preferred_proxy_for_watch method
+        if hasattr(self, 'datastore') and self.datastore:
+            try:
+                # Get preferred proxy ID if not provided
+                if not preferred_proxy_id and hasattr(self.datastore, 'get_preferred_proxy_for_watch'):
+                    # Get the watch UUID if available
+                    watch_uuid = None
+                    if hasattr(self.watch, 'get'):
+                        watch_uuid = self.watch.get('uuid')
+                    elif hasattr(self.watch, 'uuid'):
+                        watch_uuid = self.watch.uuid
+                    
+                    if watch_uuid:
+                        preferred_proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
+                
+                # Check if we have a proxy list and a valid proxy ID
+                if preferred_proxy_id and hasattr(self.datastore, 'proxy_list') and self.datastore.proxy_list:
+                    proxy_info = self.datastore.proxy_list.get(preferred_proxy_id)
+                    
+                    if proxy_info and 'url' in proxy_info:
+                        proxy_url = proxy_info.get('url')
+                        logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}'")
+                        
+                        # Parse the proxy URL to build a proxy dict for requests
+                        import urllib.parse
+                        parsed_proxy = urllib.parse.urlparse(proxy_url)
+                        proxy_type = parsed_proxy.scheme
+                        
+                        # Extract credentials if present
+                        username = None
+                        password = None
+                        if parsed_proxy.username:
+                            username = parsed_proxy.username
+                            if parsed_proxy.password:
+                                password = parsed_proxy.password
+                        
+                        # Build the proxy URL without credentials for the proxy dict
+                        netloc = parsed_proxy.netloc
+                        if '@' in netloc:
+                            netloc = netloc.split('@')[1]
+                        
+                        proxy_addr = f"{proxy_type}://{netloc}"
+                        
+                        # Create the proxy configuration
+                        proxy_config = {
+                            'http': proxy_addr,
+                            'https': proxy_addr
+                        }
+                        
+                        # Add credentials if present
+                        if username:
+                            proxy_config['username'] = username
+                            if password:
+                                proxy_config['password'] = password
+            except Exception as e:
+                # Log the error but continue without a proxy
+                logger.error(f"Error setting up proxy: {str(e)}")
+                proxy_config = None
+                proxy_url = None
+                
+        return proxy_config, proxy_url
 
     def call_browser(self, preferred_proxy_id=None):
-
+        """Fetch content using the appropriate browser/fetcher
+        
+        This method will:
+        1. Determine the appropriate fetcher to use based on watch settings
+        2. Set up proxy configuration if needed
+        3. Initialize the fetcher with the correct parameters
+        4. Configure any browser steps if needed
+        
+        Args:
+            preferred_proxy_id: Optional explicit proxy ID to use
+        """
         from requests.structures import CaseInsensitiveDict
 
         url = self.watch.link
@@ -43,8 +134,8 @@ class difference_detection_processor():
         # Requests, playwright, other browser via wss:// etc, fetch_extra_something
         prefer_fetch_backend = self.watch.get('fetch_backend', 'system')
 
-        # Proxy ID "key"
+        # Get proxy configuration
-        preferred_proxy_id = preferred_proxy_id if preferred_proxy_id else self.datastore.get_preferred_proxy_for_watch(uuid=self.watch.get('uuid'))
+        proxy_config, proxy_url = self._get_proxy_for_watch(preferred_proxy_id)
 
         # Pluggable content self.fetcher
         if not prefer_fetch_backend or prefer_fetch_backend == 'system':
@@ -82,14 +173,10 @@ class difference_detection_processor():
             # What it referenced doesnt exist, Just use a default
             fetcher_obj = getattr(content_fetchers, "html_requests")
 
-        proxy_url = None
+        # Custom browser endpoints should NOT have a proxy added
-        if preferred_proxy_id:
+        if proxy_url and prefer_fetch_backend.startswith('extra_browser_'):
-            # Custom browser endpoints should NOT have a proxy added
+            logger.debug(f"Skipping adding proxy data when custom Browser endpoint is specified.")
-            if not prefer_fetch_backend.startswith('extra_browser_'):
+            proxy_url = None
-                proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')
-                logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}' for {url}")
-            else:
-                logger.debug(f"Skipping adding proxy data when custom Browser endpoint is specified. ")
 
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
@@ -185,9 +272,9 @@ def find_sub_packages(package_name):
 
 def find_processors():
     """
-    Find all subclasses of DifferenceDetectionProcessor in the specified package.
+    Find all subclasses of DifferenceDetectionProcessor in the specified package
+    and also include processors from the plugin system.
 
-    :param package_name: The name of the package to scan for processor modules.
     :return: A list of (module, class) tuples.
     """
     package_name = "changedetectionio.processors"  # Name of the current package/module
@@ -195,6 +282,7 @@ def find_processors():
     processors = []
     sub_packages = find_sub_packages(package_name)
 
+    # Find traditional processors
     for sub_package in sub_packages:
         module_name = f"{package_name}.{sub_package}.processor"
         try:
@@ -207,6 +295,15 @@ def find_processors():
         except (ModuleNotFoundError, ImportError) as e:
             logger.warning(f"Failed to import module {module_name}: {e} (find_processors())")
 
+    # Also include processors from the plugin system
+    try:
+        from .processor_registry import get_plugin_processor_modules
+        plugin_modules = get_plugin_processor_modules()
+        if plugin_modules:
+            processors.extend(plugin_modules)
+    except (ImportError, ModuleNotFoundError) as e:
+        logger.warning(f"Failed to import plugin modules: {e} (find_processors())")
+
     return processors
 
 
@@ -223,8 +320,22 @@ def get_parent_module(module):
     return False
 
 
-
 def get_custom_watch_obj_for_processor(processor_name):
+    """
+    Get the custom watch object for a processor
+    :param processor_name: Name of the processor
+    :return: Watch class or None
+    """
+    # First, try to get the watch model from the pluggy system
+    try:
+        from .processor_registry import get_processor_watch_model
+        watch_model = get_processor_watch_model(processor_name)
+        if watch_model:
+            return watch_model
+    except Exception as e:
+        logger.warning(f"Error getting processor watch model from pluggy: {e}")
+
+    # Fall back to the traditional approach
     from changedetectionio.model import Watch
     watch_class = Watch.model
     processor_classes = find_processors()
@@ -241,14 +352,47 @@ def get_custom_watch_obj_for_processor(processor_name):
 def available_processors():
     """
     Get a list of processors by name and description for the UI elements
-    :return: A list :)
+    :return: A list of tuples (processor_name, description)
     """
-
+    # Get processors from the pluggy system
-    processor_classes = find_processors()
+    pluggy_processors = []
-
+    try:
-    available = []
+        from .processor_registry import get_all_processors
-    for package, processor_class in processor_classes:
+        pluggy_processors = get_all_processors()
-        available.append((processor_class, package.name))
+    except Exception as e:
-
+        logger.error(f"Error getting processors from pluggy: {str(e)}")
-    return available
+    
-
+    # Get processors from the traditional file-based system
+    traditional_processors = []
+    try:
+        # Let's not use find_processors() directly since it now also includes pluggy processors
+        package_name = "changedetectionio.processors"
+        sub_packages = find_sub_packages(package_name)
+        
+        for sub_package in sub_packages:
+            module_name = f"{package_name}.{sub_package}.processor"
+            try:
+                module = importlib.import_module(module_name)
+                # Get the name and description from the module if available
+                name = getattr(module, 'name', f"Traditional processor: {sub_package}")
+                description = getattr(module, 'description', sub_package)
+                traditional_processors.append((sub_package, name))
+            except (ModuleNotFoundError, ImportError, AttributeError) as e:
+                logger.warning(f"Failed to import module {module_name}: {e} (available_processors())")
+    except Exception as e:
+        logger.error(f"Error getting traditional processors: {str(e)}")
+    
+    # Combine the lists, ensuring no duplicates
+    # Pluggy processors take precedence
+    all_processors = []
+    
+    # Add all pluggy processors
+    all_processors.extend(pluggy_processors)
+    
+    # Add traditional processors that aren't already registered via pluggy
+    pluggy_processor_names = [name for name, _ in pluggy_processors]
+    for processor_class, name in traditional_processors:
+        if processor_class not in pluggy_processor_names:
+            all_processors.append((processor_class, name))
+    
+    return all_processors

changedetectionio/processors/form.py

@@ -0,0 +1,17 @@
+from wtforms import (
+    BooleanField,
+    validators,
+    RadioField
+)
+from wtforms.fields.choices import SelectField
+from wtforms.fields.form import FormField
+from wtforms.form import Form
+
+class BaseProcessorForm(Form):
+    """Base class for processor forms"""
+    
+    def extra_tab_content(self):
+        return None
+
+    def extra_form_content(self):
+        return None

changedetectionio/processors/forms.py

@@ -0,0 +1,4 @@
+"""
+Forms for processors
+"""
+from changedetectionio.forms import processor_text_json_diff_form

changedetectionio/processors/pluggy_interface.py

@@ -0,0 +1,69 @@
+import pluggy
+
+# Define the plugin namespace for processors
+PLUGIN_NAMESPACE = "changedetectionio_processors"
+
+hookspec = pluggy.HookspecMarker(PLUGIN_NAMESPACE)
+hookimpl = pluggy.HookimplMarker(PLUGIN_NAMESPACE)
+
+
+class ProcessorSpec:
+    """Hook specifications for processor plugins."""
+
+    @hookspec
+    def get_processor_name():
+        """Return the name of the processor."""
+        pass
+
+    @hookspec
+    def get_processor_description():
+        """Return the description of the processor."""
+        pass
+    
+    @hookspec
+    def get_processor_class():
+        """Return the processor class."""
+        pass
+    
+    @hookspec
+    def get_processor_form():
+        """Return the processor form class."""
+        pass
+    
+    @hookspec
+    def get_processor_watch_model():
+        """Return the watch model class for this processor (if any)."""
+        pass
+    
+    @hookspec
+    def get_display_link(url, processor_name):
+        """Return a custom display link for the given processor.
+        
+        Args:
+            url: The original URL from the watch
+            processor_name: The name of the processor
+            
+        Returns:
+            A string with the custom display link or None to use the default
+        """
+        pass
+    
+    @hookspec
+    def perform_site_check(datastore, watch_uuid):
+        """Create and return a processor instance ready to perform site check.
+        
+        Args:
+            datastore: The application datastore
+            watch_uuid: The UUID of the watch to check
+            
+        Returns:
+            A processor instance ready to perform site check
+        """
+        pass
+
+
+# Set up the plugin manager
+plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE)
+
+# Register hook specifications
+plugin_manager.add_hookspecs(ProcessorSpec)

changedetectionio/processors/processor_registry.py

@@ -0,0 +1,222 @@
+from loguru import logger
+from changedetectionio.model import Watch
+from .pluggy_interface import plugin_manager
+from typing import Dict, Any, List, Tuple, Optional, TypeVar, Type
+import functools
+
+# Import and register internal plugins
+from . import whois_plugin
+from . import test_plugin
+
+# Register plugins
+plugin_manager.register(whois_plugin)
+plugin_manager.register(test_plugin)
+
+# Load any setuptools entrypoints
+plugin_manager.load_setuptools_entrypoints("changedetectionio_processors")
+
+# Type definitions for better type hinting
+T = TypeVar('T')
+ProcessorClass = TypeVar('ProcessorClass')
+ProcessorForm = TypeVar('ProcessorForm')
+ProcessorWatchModel = TypeVar('ProcessorWatchModel')
+ProcessorInstance = TypeVar('ProcessorInstance')
+
+# Cache for plugin name mapping to improve performance
+_plugin_name_map: Dict[str, Any] = {}
+
+def register_plugin(plugin_module):
+    """Register a processor plugin"""
+    plugin_manager.register(plugin_module)
+    # Clear the plugin name map cache when a new plugin is registered
+    global _plugin_name_map
+    _plugin_name_map = {}
+
+def _get_plugin_name_map() -> Dict[str, Any]:
+    """Get a mapping of processor names to plugins
+    :return: Dictionary mapping processor names to plugins
+    """
+    global _plugin_name_map
+    
+    # Return cached map if available
+    if _plugin_name_map:
+        return _plugin_name_map
+    
+    # Build the map
+    result = {}
+    
+    # Get all plugins from the plugin manager
+    all_plugins = list(plugin_manager.get_plugins())
+    
+    # First register known internal plugins by name for reliability
+    known_plugins = {
+        'whois': whois_plugin,
+        'test': test_plugin
+    }
+    
+    for name, plugin in known_plugins.items():
+        if plugin in all_plugins:
+            result[name] = plugin
+    
+    # Then process remaining plugins through the hook system
+    for plugin in all_plugins:
+        if plugin in known_plugins.values():
+            continue  # Skip plugins we've already registered
+            
+        try:
+            # Get the processor name from this plugin
+            name_results = plugin_manager.hook.get_processor_name(plugin=plugin)
+            
+            if name_results:
+                plugin_name = name_results[0]
+                
+                # Check for name collisions
+                if plugin_name in result:
+                    logger.warning(f"Plugin name collision: '{plugin_name}' is already registered")
+                    continue
+                    
+                result[plugin_name] = plugin
+        except Exception as e:
+            logger.error(f"Error getting processor name from plugin: {str(e)}")
+    
+    # Cache the map
+    _plugin_name_map = result
+    return result
+
+def _get_plugin_by_name(processor_name: str) -> Optional[Any]:
+    """Get a plugin by its processor name
+    :param processor_name: Name of the processor
+    :return: Plugin object or None
+    """
+    return _get_plugin_name_map().get(processor_name)
+
+def _call_hook_for_plugin(plugin: Any, hook_name: str, default_value: T = None, **kwargs) -> Optional[T]:
+    """Call a hook for a specific plugin and handle exceptions
+    :param plugin: The plugin to call the hook for
+    :param hook_name: Name of the hook to call
+    :param default_value: Default value to return if the hook call fails
+    :param kwargs: Additional arguments to pass to the hook
+    :return: Result of the hook call or default value
+    """
+    if not plugin:
+        return default_value
+    
+    try:
+        hook = getattr(plugin_manager.hook, hook_name)
+        results = hook(plugin=plugin, **kwargs)
+        
+        if results:
+            return results[0]
+    except Exception as e:
+        logger.error(f"Error calling {hook_name} for plugin: {str(e)}")
+    
+    return default_value
+
+def get_all_processors() -> List[Tuple[str, str]]:
+    """Get all processors
+    :return: List of tuples (processor_name, processor_description)
+    """
+    processors = []
+    
+    for processor_name, plugin in _get_plugin_name_map().items():
+        description = _call_hook_for_plugin(plugin, 'get_processor_description')
+        if description:
+            processors.append((processor_name, description))
+    
+    return processors
+
+def get_processor_class(processor_name: str) -> Optional[Type[ProcessorClass]]:
+    """Get processor class by name
+    :param processor_name: Name of the processor
+    :return: Processor class or None
+    """
+    plugin = _get_plugin_by_name(processor_name)
+    return _call_hook_for_plugin(plugin, 'get_processor_class')
+
+def get_processor_form(processor_name: str) -> Optional[Type[ProcessorForm]]:
+    """Get processor form by name
+    :param processor_name: Name of the processor
+    :return: Processor form class or None
+    """
+    plugin = _get_plugin_by_name(processor_name)
+    return _call_hook_for_plugin(plugin, 'get_processor_form')
+
+def get_processor_watch_model(processor_name: str) -> Type[ProcessorWatchModel]:
+    """Get processor watch model by name
+    :param processor_name: Name of the processor
+    :return: Watch model class or default Watch model
+    """
+    plugin = _get_plugin_by_name(processor_name)
+    return _call_hook_for_plugin(plugin, 'get_processor_watch_model', default_value=Watch.model)
+
+def get_processor_site_check(processor_name: str, datastore: Any, watch_uuid: str) -> Optional[ProcessorInstance]:
+    """Get a processor instance ready to perform site check
+    :param processor_name: Name of the processor
+    :param datastore: The application datastore
+    :param watch_uuid: The UUID of the watch to check
+    :return: A processor instance ready to perform site check, or None
+    """
+    plugin = _get_plugin_by_name(processor_name)
+    if not plugin:
+        return None
+    
+    # Try to get the perform_site_check implementation
+    try:
+        processor = _call_hook_for_plugin(
+            plugin, 
+            'perform_site_check', 
+            datastore=datastore, 
+            watch_uuid=watch_uuid
+        )
+        if processor:
+            return processor
+        
+        # If no perform_site_check hook implementation, try getting the class and instantiating it
+        processor_class = _call_hook_for_plugin(plugin, 'get_processor_class')
+        if processor_class:
+            return processor_class(datastore=datastore, watch_uuid=watch_uuid)
+    except Exception as e:
+        logger.error(f"Error getting processor site check for {processor_name}: {str(e)}")
+    
+    return None
+
+def get_display_link(url: str, processor_name: str) -> Optional[str]:
+    """Get a custom display link for the given processor
+    :param url: The original URL from the watch
+    :param processor_name: Name of the processor
+    :return: A string with the custom display link or None to use the default
+    """
+    plugin = _get_plugin_by_name(processor_name)
+    return _call_hook_for_plugin(
+        plugin, 
+        'get_display_link', 
+        url=url, 
+        processor_name=processor_name
+    )
+
+def get_plugin_processor_modules() -> List[Tuple[Any, str]]:
+    """Get processor modules for all plugins that can be used with the find_processors function
+    
+    This function adapts pluggy plugins to be compatible with the traditional find_processors system
+    
+    :return: A list of (module, processor_name) tuples
+    """
+    result = []
+    
+    # Import base modules once to avoid repeated imports
+    from changedetectionio.processors.text_json_diff import processor as text_json_diff_processor
+
+    # For each plugin, map to a suitable module for find_processors
+    for processor_name, plugin in _get_plugin_name_map().items():
+        try:
+            processor_class = _call_hook_for_plugin(plugin, 'get_processor_class')
+            
+            if processor_class:
+                # Check if this processor extends the text_json_diff processor
+                base_class_name = str(processor_class.__bases__[0].__name__)
+                if base_class_name == 'perform_site_check' or 'TextJsonDiffProcessor' in base_class_name:
+                    result.append((text_json_diff_processor, processor_name))
+        except Exception as e:
+            logger.error(f"Error mapping processor module for {processor_name}: {str(e)}")
+    
+    return result

changedetectionio/processors/text_json_diff/__init__.py

@@ -28,13 +28,13 @@ def _task(watch, update_handler):
     return text_after_filter
 
 
-def prepare_filter_prevew(datastore, watch_uuid):
+def prepare_filter_prevew(datastore, watch_uuid, form_data):
     '''Used by @app.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])'''
     from changedetectionio import forms, html_tools
     from changedetectionio.model.Watch import model as watch_model
     from concurrent.futures import ProcessPoolExecutor
     from copy import deepcopy
-    from flask import request, jsonify
+    from flask import request
     import brotli
     import importlib
     import os
@@ -50,12 +50,12 @@ def prepare_filter_prevew(datastore, watch_uuid):
 
     if tmp_watch and tmp_watch.history and os.path.isdir(tmp_watch.watch_data_dir):
         # Splice in the temporary stuff from the form
-        form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None,
+        form = forms.processor_text_json_diff_form(formdata=form_data if request.method == 'POST' else None,
-                                                   data=request.form
+                                                   data=form_data
                                                    )
 
         # Only update vars that came in via the AJAX post
-        p = {k: v for k, v in form.data.items() if k in request.form.keys()}
+        p = {k: v for k, v in form.data.items() if k in form_data.keys()}
         tmp_watch.update(p)
         blank_watch_no_filters = watch_model()
         blank_watch_no_filters['url'] = tmp_watch.get('url')
@@ -103,13 +103,12 @@ def prepare_filter_prevew(datastore, watch_uuid):
 
     logger.trace(f"Parsed in {time.time() - now:.3f}s")
 
-    return jsonify(
+    return ({
-        {
             'after_filter': text_after_filter,
             'before_filter': text_before_filter.decode('utf-8') if isinstance(text_before_filter, bytes) else text_before_filter,
             'duration': time.time() - now,
             'trigger_line_numbers': trigger_line_numbers,
             'ignore_line_numbers': ignore_line_numbers,
-        }
+        })
-    )
+
 

changedetectionio/processors/text_json_diff/processor.py

@@ -6,6 +6,7 @@ import os
 import re
 import urllib3
 
+from changedetectionio.conditions import execute_ruleset_against_all_plugins
 from changedetectionio.processors import difference_detection_processor
 from changedetectionio.html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text, TRANSLATE_WHITESPACE_TABLE
 from changedetectionio import html_tools, content_fetchers
@@ -331,6 +332,16 @@ class perform_site_check(difference_detection_processor):
             if result:
                 blocked = True
 
+        # And check if 'conditions' will let this pass through
+        if watch.get('conditions') and watch.get('conditions_match_logic'):
+            if not execute_ruleset_against_all_plugins(current_watch_uuid=watch.get('uuid'),
+                                                application_datastruct=self.datastore.data,
+                                                ephemeral_data={
+                                                    'text': stripped_text_from_html
+                                                }
+                                                ):
+                # Conditions say "Condition not met" so we block it.
+                blocked = True
 
         # Looks like something changed, but did it match all the rules?
         if blocked:

changedetectionio/processors/whois_plugin.py

@@ -0,0 +1,169 @@
+from loguru import logger
+import re
+import urllib.parse
+from .pluggy_interface import hookimpl
+from requests.structures import CaseInsensitiveDict
+from changedetectionio.content_fetchers.base import Fetcher
+
+# Import the text_json_diff processor
+from changedetectionio.processors.text_json_diff.processor import perform_site_check as TextJsonDiffProcessor
+
+# WHOIS Processor implementation that extends TextJsonDiffProcessor
+class WhoisProcessor(TextJsonDiffProcessor):
+    
+    def _extract_domain_from_url(self, url):
+        """Extract domain from URL, removing www. prefix if present"""
+        parsed_url = urllib.parse.urlparse(url)
+        domain = parsed_url.netloc
+        
+        # Remove www. prefix if present
+        domain = re.sub(r'^www\.', '', domain)
+        
+        return domain
+    
+    def call_browser(self, preferred_proxy_id=None):
+        """Override call_browser to perform WHOIS lookup instead of using a browser
+        
+        Note: The python-whois library doesn't directly support proxies. For real proxy support,
+        we would need to implement a custom socket connection that routes through the proxy.
+        This is a TODO for a future enhancement.
+        """
+        # Initialize a basic fetcher - this is used by the parent class
+        self.fetcher = Fetcher()
+        
+        # Extract URL from watch
+        url = self.watch.link
+        
+        # Check for file:// access
+        if re.search(r'^file:', url.strip(), re.IGNORECASE):
+            if not self.datastore.data.get('settings', {}).get('application', {}).get('allow_file_uri', False):
+                raise Exception("file:// type access is denied for security reasons.")
+        
+        # Extract domain from URL
+        domain = self._extract_domain_from_url(url)
+        
+        # Ensure we have a valid domain
+        if not domain:
+            error_msg = f"Could not extract domain from URL: '{url}'"
+            self.fetcher.content = error_msg
+            self.fetcher.status_code = 400
+            logger.error(error_msg)
+            return
+        
+        # Get proxy configuration using the common method from parent class
+        proxy_config, proxy_url = super()._get_proxy_for_watch(preferred_proxy_id)
+        
+        try:
+            # Use python-whois to get domain information
+            import whois
+            
+            # If we have proxy config, use it for the WHOIS lookup
+            # Note: The python-whois library doesn't directly support proxies,
+            # but we can implement proxy support if necessary using custom socket code
+            if proxy_config:
+                # For now, just log that we would use a proxy
+                logger.info(f"Using proxy for WHOIS lookup: {proxy_config}")
+            
+            # Perform the WHOIS lookup
+            whois_info = whois.whois(domain)
+            
+            # Convert whois_info object to text
+            if hasattr(whois_info, 'text'):
+                # Some whois implementations store raw text in .text attribute
+                whois_text = whois_info.text
+            else:
+                # Otherwise, format it nicely as key-value pairs
+                whois_text = f"WHOIS Information for domain: {domain}\n\n"
+                for key, value in whois_info.items():
+                    if value:
+                        whois_text += f"{key}: {value}\n"
+            
+            # Set the content and status for the fetcher
+            self.fetcher.content = whois_text
+            self.fetcher.status_code = 200
+            
+            # Setup headers dictionary for the fetcher
+            self.fetcher.headers = CaseInsensitiveDict({
+                'content-type': 'text/plain',
+                'server': 'whois-processor'
+            })
+            
+            # Add getters for headers
+            self.fetcher.get_all_headers = lambda: self.fetcher.headers
+            self.fetcher.get_last_status_code = lambda: self.fetcher.status_code
+            
+            # Implement necessary methods
+            self.fetcher.quit = lambda: None
+            
+        except Exception as e:
+            error_msg = f"Error fetching WHOIS data for domain {domain}: {str(e)}"
+            self.fetcher.content = error_msg
+            self.fetcher.status_code = 500
+            self.fetcher.headers = CaseInsensitiveDict({
+                'content-type': 'text/plain',
+                'server': 'whois-processor'
+            })
+            self.fetcher.get_all_headers = lambda: self.fetcher.headers
+            self.fetcher.get_last_status_code = lambda: self.fetcher.status_code
+            self.fetcher.quit = lambda: None
+            logger.error(error_msg)
+
+        return
+    
+    def run_changedetection(self, watch):
+        """Use the parent's run_changedetection which will use our overridden call_browser method"""
+        try:
+            # Let the parent class handle everything now that we've overridden call_browser
+            changed_detected, update_obj, filtered_text = super().run_changedetection(watch)
+            return changed_detected, update_obj, filtered_text
+            
+        except Exception as e:
+            error_msg = f"Error in WHOIS processor: {str(e)}"
+            update_obj = {'last_notification_error': False, 'last_error': error_msg}
+            logger.error(error_msg)
+            return False, update_obj, error_msg.encode('utf-8')
+
+    @staticmethod
+    def perform_site_check(datastore, watch_uuid):
+        """Factory method to create a WhoisProcessor instance - for compatibility with legacy code"""
+        processor = WhoisProcessor(datastore=datastore, watch_uuid=watch_uuid)
+        return processor
+
+@hookimpl
+def perform_site_check(datastore, watch_uuid):
+    """Create and return a processor instance ready to perform site check"""
+    return WhoisProcessor(datastore=datastore, watch_uuid=watch_uuid)
+
+@hookimpl(trylast=True)  # Use trylast to ensure this runs last in case of conflicts
+def get_processor_name():
+    """Return the name of this processor"""
+    from loguru import logger
+    logger.debug("whois_plugin.get_processor_name() called")
+    return "whois"
+
+@hookimpl
+def get_processor_description():
+    """Return the description of this processor"""
+    return "WHOIS Domain Information Changes Detector"
+
+@hookimpl
+def get_processor_class():
+    """Return the processor class"""
+    return WhoisProcessor
+
+@hookimpl
+def get_processor_form():
+    """Return the processor form class"""
+    # Import here to avoid circular imports
+    try:
+        from changedetectionio.forms import processor_text_json_diff_form
+        return processor_text_json_diff_form
+    except Exception as e:
+        from loguru import logger
+        logger.error(f"Error importing form for whois plugin: {str(e)}")
+        return None
+
+@hookimpl
+def get_processor_watch_model():
+    """Return the watch model class for this processor"""
+    return None  # Use default watch model

changedetectionio/static/images/copy.svg

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <svg
    version="1.1"
-   id="Layer_1"
+   id="copy"
    x="0px"
    y="0px"
    viewBox="0 0 115.77 122.88"

changedetectionio/static/images/email.svg

@@ -6,7 +6,7 @@
    height="7.5005589"
    width="11.248507"
    version="1.1"
-   id="Layer_1"
+   id="email"
    viewBox="0 0 7.1975545 4.7993639"
    xml:space="preserve"
    xmlns="http://www.w3.org/2000/svg"

changedetectionio/static/images/schedule.svg

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <svg
    version="1.1"
-   id="Layer_1"
+   id="schedule"
    x="0px"
    y="0px"
    viewBox="0 0 661.20001 665.40002"

changedetectionio/static/js/browser-steps.js

@@ -221,7 +221,7 @@ $(document).ready(function () {
                     // If you switch to "Click X,y" after an element here is setup, it will give the last co-ords anyway
                     //if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
                         $('select', first_available).val('Click element').change();
-                        $('input[type=text]', first_available).first().val(x['xpath']);
+                        $('input[type=text]', first_available).first().val(x['xpath']).focus();
                         found_something = true;
                     //}
                 }
@@ -305,7 +305,7 @@ $(document).ready(function () {
 
         if ($(this).val() === 'Click X,Y' && last_click_xy['x'] > 0 && $(elem_value).val().length === 0) {
             // @todo handle scale
-            $(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']);
+            $(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']).focus();
         }
     }).change();
 

changedetectionio/static/js/conditions.js

@@ -0,0 +1,150 @@
+$(document).ready(function () {
+    // Function to set up button event handlers
+    function setupButtonHandlers() {
+        // Unbind existing handlers first to prevent duplicates
+        $(".addRuleRow, .removeRuleRow, .verifyRuleRow").off("click");
+        
+        // Add row button handler
+        $(".addRuleRow").on("click", function(e) {
+            e.preventDefault();
+            
+            let currentRow = $(this).closest("tr");
+            
+            // Clone without events
+            let newRow = currentRow.clone(false);
+            
+            // Reset input values in the cloned row
+            newRow.find("input").val("");
+            newRow.find("select").prop("selectedIndex", 0);
+            
+            // Insert the new row after the current one
+            currentRow.after(newRow);
+            
+            // Reindex all rows
+            reindexRules();
+        });
+        
+        // Remove row button handler
+        $(".removeRuleRow").on("click", function(e) {
+            e.preventDefault();
+            
+            // Only remove if there's more than one row
+            if ($("#rulesTable tbody tr").length > 1) {
+                $(this).closest("tr").remove();
+                reindexRules();
+            }
+        });
+        
+        // Verify rule button handler
+        $(".verifyRuleRow").on("click", function(e) {
+            e.preventDefault();
+            
+            let row = $(this).closest("tr");
+            let field = row.find("select[name$='field']").val();
+            let operator = row.find("select[name$='operator']").val();
+            let value = row.find("input[name$='value']").val();
+            
+            // Validate that all fields are filled
+            if (!field || field === "None" || !operator || operator === "None" || !value) {
+                alert("Please fill in all fields (Field, Operator, and Value) before verifying.");
+                return;
+            }
+
+            
+            // Create a rule object
+            const rule = {
+                field: field,
+                operator: operator,
+                value: value
+            };
+            
+            // Show a spinner or some indication that verification is in progress
+            const $button = $(this);
+            const originalHTML = $button.html();
+            $button.html("⌛").prop("disabled", true);
+            
+            // Collect form data - similar to request_textpreview_update() in watch-settings.js
+            let formData = new FormData();
+            $('#edit-text-filter textarea, #edit-text-filter input').each(function() {
+                const $element = $(this);
+                const name = $element.attr('name');
+                if (name) {
+                    if ($element.is(':checkbox')) {
+                        formData.append(name, $element.is(':checked') ? $element.val() : false);
+                    } else {
+                        formData.append(name, $element.val());
+                    }
+                }
+            });
+            
+            // Also collect select values
+            $('#edit-text-filter select').each(function() {
+                const $element = $(this);
+                const name = $element.attr('name');
+                if (name) {
+                    formData.append(name, $element.val());
+                }
+            });
+
+
+            // Send the request to verify the rule
+            $.ajax({
+                url: verify_condition_rule_url+"?"+ new URLSearchParams({ rule: JSON.stringify(rule) }).toString(),
+                type: "POST",
+                data: formData,
+                processData: false, // Prevent jQuery from converting FormData to a string
+                contentType: false, // Let the browser set the correct content type
+                success: function (response) {
+                    if (response.status === "success") {
+                        if (response.result) {
+                            alert("✅ Condition PASSES verification against current snapshot!");
+                        } else {
+                            alert("❌ Condition FAILS verification against current snapshot.");
+                        }
+                    } else {
+                        alert("Error: " + response.message);
+                    }
+                    $button.html(originalHTML).prop("disabled", false);
+                },
+                error: function (xhr) {
+                    let errorMsg = "Error verifying condition.";
+                    if (xhr.responseJSON && xhr.responseJSON.message) {
+                        errorMsg = xhr.responseJSON.message;
+                    }
+                    alert(errorMsg);
+                    $button.html(originalHTML).prop("disabled", false);
+                }
+            });
+        });
+    }
+
+    // Function to reindex form elements and re-setup event handlers
+    function reindexRules() {
+        // Unbind all button handlers first
+        $(".addRuleRow, .removeRuleRow, .verifyRuleRow").off("click");
+        
+        // Reindex all form elements
+        $("#rulesTable tbody tr").each(function(index) {
+            $(this).find("select, input").each(function() {
+                let oldName = $(this).attr("name");
+                let oldId = $(this).attr("id");
+
+                if (oldName) {
+                    let newName = oldName.replace(/\d+/, index);
+                    $(this).attr("name", newName);
+                }
+
+                if (oldId) {
+                    let newId = oldId.replace(/\d+/, index);
+                    $(this).attr("id", newId);
+                }
+            });
+        });
+        
+        // Reattach event handlers after reindexing
+        setupButtonHandlers();
+    }
+
+    // Initial setup of button handlers
+    setupButtonHandlers();
+});

changedetectionio/static/js/notifications.js

@@ -1,42 +1,52 @@
-$(document).ready(function() {
+$(document).ready(function () {
 
-  $('#add-email-helper').click(function (e) {
+    $('#add-email-helper').click(function (e) {
-    e.preventDefault();
+        e.preventDefault();
-    email = prompt("Destination email");
+        email = prompt("Destination email");
-    if(email) {
+        if (email) {
-      var n = $(".notification-urls");
+            var n = $(".notification-urls");
-      var p=email_notification_prefix;
+            var p = email_notification_prefix;
-      $(n).val( $.trim( $(n).val() )+"\n"+email_notification_prefix+email );
+            $(n).val($.trim($(n).val()) + "\n" + email_notification_prefix + email);
-    }
-  });
-
-  $('#send-test-notification').click(function (e) {
-    e.preventDefault();
-
-    data = {
-      notification_body: $('#notification_body').val(),
-      notification_format: $('#notification_format').val(),
-      notification_title: $('#notification_title').val(),
-      notification_urls: $('.notification-urls').val(),
-      tags: $('#tags').val(),
-      window_url: window.location.href,
-    }
-
-
-    $.ajax({
-      type: "POST",
-      url: notification_base_url,
-      data : data,
-        statusCode: {
-        400: function(data) {
-          // More than likely the CSRF token was lost when the server restarted
-          alert(data.responseText);
         }
-      }
+    });
-    }).done(function(data){
+
-      console.log(data);
+    $('#send-test-notification').click(function (e) {
-      alert(data);
+        e.preventDefault();
-    })
+
-  });
+        data = {
+            notification_body: $('#notification_body').val(),
+            notification_format: $('#notification_format').val(),
+            notification_title: $('#notification_title').val(),
+            notification_urls: $('.notification-urls').val(),
+            tags: $('#tags').val(),
+            window_url: window.location.href,
+        }
+
+        $('.notifications-wrapper .spinner').fadeIn();
+        $('#notification-test-log').show();
+        $.ajax({
+            type: "POST",
+            url: notification_base_url,
+            data: data,
+            statusCode: {
+                400: function (data) {
+                    $("#notification-test-log>span").text(data.responseText);
+                },
+            }
+        }).done(function (data) {
+            $("#notification-test-log>span").text(data);
+        }).fail(function (jqXHR, textStatus, errorThrown) {
+            // Handle connection refused or other errors
+            if (textStatus === "error" && errorThrown === "") {
+                console.error("Connection refused or server unreachable");
+                $("#notification-test-log>span").text("Error: Connection refused or server is unreachable.");
+            } else {
+                console.error("Error:", textStatus, errorThrown);
+                $("#notification-test-log>span").text("An error occurred: " + textStatus);
+            }
+        }).always(function () {
+            $('.notifications-wrapper .spinner').hide();
+        })
+    });
 });
 

changedetectionio/static/js/tabs.js

@@ -26,7 +26,6 @@ function set_active_tab() {
     if (tab.length) {
         tab[0].parentElement.className = "active";
     }
-
 }
 
 function focus_error_tab() {

changedetectionio/static/styles/scss/parts/_browser-steps.scss

@@ -40,19 +40,22 @@
   }
 }
 
+@media only screen and (min-width: 760px) {
 
-#browser-steps .flex-wrapper {
+  #browser-steps .flex-wrapper {
-  display: flex;
+    display: flex;
-  flex-flow: row;
+    flex-flow: row;
-  height: 70vh;
+    height: 70vh;
-  font-size: 80%;
+    font-size: 80%;
-  #browser-steps-ui {
-    flex-grow: 1;      /* Allow it to grow and fill the available space */
-    flex-shrink: 1;    /* Allow it to shrink if needed */
-    flex-basis: 0;     /* Start with 0 base width so it stretches as much as possible */
-    background-color: #eee;
-    border-radius: 5px;
 
+    #browser-steps-ui {
+      flex-grow: 1; /* Allow it to grow and fill the available space */
+      flex-shrink: 1; /* Allow it to shrink if needed */
+      flex-basis: 0; /* Start with 0 base width so it stretches as much as possible */
+      background-color: #eee;
+      border-radius: 5px;
+
+    }
   }
 
   #browser-steps-fieldlist {
@@ -63,15 +66,21 @@
     padding-left: 1rem;
     overflow-y: scroll;
   }
+
+  /*  this is duplicate :( */
+  #browsersteps-selector-wrapper {
+    height: 100% !important;
+  }
 }
 
 /*  this is duplicate :( */
 #browsersteps-selector-wrapper {
-  height: 100%;
+
   width: 100%;
   overflow-y: scroll;
   position: relative;
-  //width: 100%;
+  height: 80vh;
+
   > img {
     position: absolute;
     max-width: 100%;
@@ -91,7 +100,6 @@
     left: 50%;
     top: 50%;
     transform: translate(-50%, -50%);
-    margin-left: -40px;
     z-index: 100;
     max-width: 350px;
     text-align: center;

changedetectionio/static/styles/scss/parts/_edit.scss

@@ -0,0 +1,9 @@
+ul#conditions_match_logic {
+    list-style: none;
+  input, label, li {
+    display: inline-block;
+  }
+  li {
+    padding-right: 1em;
+  }
+}

changedetectionio/static/styles/scss/styles.scss

@@ -13,6 +13,7 @@
 @import "parts/_menu";
 @import "parts/_love";
 @import "parts/preview_text_filter";
+@import "parts/_edit";
 
 body {
   color: var(--color-text);
@@ -380,7 +381,15 @@ a.pure-button-selected {
 }
 
 .notifications-wrapper {
-  padding: 0.5rem 0 1rem 0;
+  padding-top: 0.5rem;
+  #notification-test-log {
+    padding-top: 1rem;
+    white-space: pre-wrap;
+    word-break: break-word;
+    overflow-wrap: break-word;
+    max-width: 100%;
+    box-sizing: border-box;
+  }
 }
 
 label {

changedetectionio/static/styles/styles.css

@@ -46,21 +46,22 @@
     #browser_steps li > label {
       display: none; }
 
-#browser-steps .flex-wrapper {
+@media only screen and (min-width: 760px) {
-  display: flex;
+  #browser-steps .flex-wrapper {
-  flex-flow: row;
+    display: flex;
-  height: 70vh;
+    flex-flow: row;
-  font-size: 80%; }
+    height: 70vh;
-  #browser-steps .flex-wrapper #browser-steps-ui {
+    font-size: 80%; }
-    flex-grow: 1;
+    #browser-steps .flex-wrapper #browser-steps-ui {
-    /* Allow it to grow and fill the available space */
+      flex-grow: 1;
-    flex-shrink: 1;
+      /* Allow it to grow and fill the available space */
-    /* Allow it to shrink if needed */
+      flex-shrink: 1;
-    flex-basis: 0;
+      /* Allow it to shrink if needed */
-    /* Start with 0 base width so it stretches as much as possible */
+      flex-basis: 0;
-    background-color: #eee;
+      /* Start with 0 base width so it stretches as much as possible */
-    border-radius: 5px; }
+      background-color: #eee;
-  #browser-steps .flex-wrapper #browser-steps-fieldlist {
+      border-radius: 5px; }
+  #browser-steps-fieldlist {
     flex-grow: 0;
     /* Don't allow it to grow */
     flex-shrink: 0;
@@ -71,13 +72,16 @@
     /* Set a max width to prevent overflow */
     padding-left: 1rem;
     overflow-y: scroll; }
+  /*  this is duplicate :( */
+  #browsersteps-selector-wrapper {
+    height: 100% !important; } }
 
 /*  this is duplicate :( */
 #browsersteps-selector-wrapper {
-  height: 100%;
   width: 100%;
   overflow-y: scroll;
   position: relative;
+  height: 80vh;
   /* nice tall skinny one */ }
   #browsersteps-selector-wrapper > img {
     position: absolute;
@@ -92,7 +96,6 @@
     left: 50%;
     top: 50%;
     transform: translate(-50%, -50%);
-    margin-left: -40px;
     z-index: 100;
     max-width: 350px;
     text-align: center; }
@@ -520,6 +523,13 @@ body.preview-text-enabled {
   z-index: 3;
   box-shadow: 1px 1px 4px var(--color-shadow-jump); }
 
+ul#conditions_match_logic {
+  list-style: none; }
+  ul#conditions_match_logic input, ul#conditions_match_logic label, ul#conditions_match_logic li {
+    display: inline-block; }
+  ul#conditions_match_logic li {
+    padding-right: 1em; }
+
 body {
   color: var(--color-text);
   background: var(--color-background-page);
@@ -780,7 +790,14 @@ a.pure-button-selected {
   cursor: pointer; }
 
 .notifications-wrapper {
-  padding: 0.5rem 0 1rem 0; }
+  padding-top: 0.5rem; }
+  .notifications-wrapper #notification-test-log {
+    padding-top: 1rem;
+    white-space: pre-wrap;
+    word-break: break-word;
+    overflow-wrap: break-word;
+    max-width: 100%;
+    box-sizing: border-box; }
 
 label:hover {
   cursor: pointer; }

changedetectionio/templates/_common_fields.html

@@ -12,23 +12,25 @@
                             }}
                             <div class="pure-form-message-inline">
                                 <p>
-                                <strong>Tip:</strong> Use <a target=_new href="https://github.com/caronc/apprise">AppRise Notification URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.<br>
+                                <strong>Tip:</strong> Use <a target="newwindow" href="https://github.com/caronc/apprise">AppRise Notification URLs</a> for notification to just about any service! <i><a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.<br>
 </p>
                                 <div data-target="#advanced-help-notifications" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</div>
                                 <ul style="display: none" id="advanced-help-notifications">
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
+                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
+                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
+                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
                                 <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>) <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes#postposts">more help here</a></li>
                                   <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                               </ul>
                             </div>
                             <div class="notifications-wrapper">
-                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a>
+                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a> <div class="spinner"  style="display: none;"></div>
                             {% if emailprefix %}
                               <a id="add-email-helper" class="pure-button button-secondary button-xsmall" >Add email <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='email.svg')}}" alt="Add an email address"> </a>
                             {% endif %}
                               <a href="{{url_for('notification_logs')}}" class="pure-button button-secondary button-xsmall" >Notification debug logs</a>
+                              <br>
+                                <div id="notification-test-log" style="display: none;"><span class="pure-form-message-inline">Processing..</span></div>
                             </div>
                         </div>
                         <div id="notification-customisation" class="pure-control-group">
@@ -38,7 +40,7 @@
                             </div>
                             <div class="pure-control-group">
                                 {{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }}
-                                <span class="pure-form-message-inline">Body for all notifications &dash; You can use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
+                                <span class="pure-form-message-inline">Body for all notifications &dash; You can use <a target="newwindow" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
                                 </span>
 
                             </div>
@@ -124,7 +126,7 @@
                                 <div class="pure-form-message-inline">
                                     <p>
 									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. <br>
-                                    For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
+                                    For example, an addition or removal could be perceived as a change in some cases. <a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
                                     </p>
                                     <p>
                                         For JSON payloads, use <strong>|tojson</strong> without quotes for automatic escaping, for example - <code>{ "name": {{ '{{ watch_title|tojson }}' }} }</code>

changedetectionio/templates/_helpers.html

@@ -61,6 +61,55 @@
   {{ field(**kwargs)|safe }}
 {% endmacro %}
 
+{% macro render_fieldlist_of_formfields_as_table(fieldlist, table_id="rulesTable") %}
+  <table class="fieldlist_formfields pure-table" id="{{ table_id }}">
+    <thead>
+      <tr>
+        {% for subfield in fieldlist[0] %}
+          <th>{{ subfield.label }}</th>
+        {% endfor %}
+        <th>Actions</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for form_row in fieldlist %}
+        <tr {% if form_row.errors %} class="error-row" {% endif %}>
+          {% for subfield in form_row %}
+            <td>
+              {{ subfield()|safe }}
+              {% if subfield.errors %}
+                <ul class="errors">
+                  {% for error in subfield.errors %}
+                    <li class="error">{{ error }}</li>
+                  {% endfor %}
+                </ul>
+              {% endif %}
+            </td>
+          {% endfor %}
+          <td>
+            <button type="button" class="addRuleRow">+</button>
+            <button type="button" class="removeRuleRow">-</button>
+            <button type="button" class="verifyRuleRow" title="Verify this rule against current snapshot">✓</button>
+          </td>
+        </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+{% endmacro %}
+
+
+{% macro playwright_warning() %}
+    <p><strong>Error - Playwright support for Chrome based fetching is not enabled.</strong> Alternatively try our <a href="https://changedetection.io">very affordable subscription based service which has all this setup for you</a>.</p>
+    <p>You may need to <a href="https://github.com/dgtlmoon/changedetection.io/blob/09ebc6ec6338545bdd694dc6eee57f2e9d2b8075/docker-compose.yml#L31">Enable playwright environment variable</a> and uncomment the <strong>sockpuppetbrowser</strong> in the <a href="https://github.com/dgtlmoon/changedetection.io/blob/master/docker-compose.yml">docker-compose.yml</a> file.</p>
+    <br>
+    <p>(Also Selenium/WebDriver can not extract full page screenshots reliably so Playwright is recommended here)</p>
+
+{% endmacro %}
+
+{% macro only_webdriver_type_watches_warning() %}
+    <p><strong>Sorry, this functionality only works with Playwright/Chrome enabled watches.<br>You need to <a href="#request">Set the fetch method to Playwright/Chrome mode and resave</a> and have the Playwright connection enabled.</strong></p><br>
+{% endmacro %}
+
 {% macro render_time_schedule_form(form, available_timezones, timezone_default_config) %}
     <style>
     .day-schedule *, .day-schedule select {

changedetectionio/templates/base.html

@@ -159,7 +159,7 @@
                     <a id="chrome-extension-link"
                        title="Try our new Chrome Extension!"
                        href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                        <img src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}">
+                        <img alt="Chrome store icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}">
                         Chrome Webstore
                     </a>
                 </p>

changedetectionio/templates/edit.html

@@ -1,11 +1,14 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, playwright_warning, only_webdriver_type_watches_warning, render_fieldlist_of_formfields_as_table %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='scheduler.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='conditions.js')}}" defer></script>
+
+
 <script>
     const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
@@ -40,17 +43,17 @@
 
     <div class="tabs collapsable">
         <ul>
-            <li class="tab" id=""><a href="#general">General</a></li>
+            <li class="tab"><a href="#general">General</a></li>
             <li class="tab"><a href="#request">Request</a></li>
             {% if extra_tab_content %}
             <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
             {% endif %}
-            {% if playwright_enabled %}
             <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
-            {% endif %}
+        <!-- should goto extra forms? -->
             {% if watch['processor'] == 'text_json_diff' %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
             <li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
+            <li class="tab" id="conditions-tab"><a href="#conditions">Conditions</a></li>
             {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
             <li class="tab"><a href="#stats">Stats</a></li>
@@ -199,8 +202,9 @@ Math: {{ 1 + 1 }}") }}
                     </div>
             </fieldset>
             </div>
-            {% if playwright_enabled %}
+
             <div class="tab-pane-inner" id="browser-steps">
+            {% if playwright_enabled and watch_uses_webdriver %}
                 <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
                 <fieldset>
                     <div class="pure-control-group">
@@ -224,7 +228,7 @@ Math: {{ 1 + 1 }}") }}
                                     <span class="loader" >
                                         <span id="browsersteps-click-start">
                                             <h2 >Click here to Start</h2>
-                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="Layer_1"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
+                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="start"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
                                             Please allow 10-15 seconds for the browser to connect.<br>
                                         </span>
                                         <div class="spinner"  style="display: none;"></div>
@@ -234,21 +238,31 @@ Math: {{ 1 + 1 }}") }}
                                 </div>
                             </div>
                             <div id="browser-steps-fieldlist" >
-                                <span id="browser-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target=_new href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
+                                <span id="browser-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
                                 {{ render_field(form.browser_steps) }}
                             </div>
                         </div>
                     </div>
                 </fieldset>
+                {% else %}
+                    <span class="pure-form-message-inline">
+                        {% if not watch_uses_webdriver %}
+                            {{ only_webdriver_type_watches_warning() }}
+                        {% endif %}
+                        {%  if not playwright_enabled %}
+                            {{ playwright_warning() }}
+                        {% endif %}
+                    </span>
+                {% endif %}
             </div>
-            {% endif %}
+
 
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div  class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_muted) }}
                     </div>
-                    {% if is_html_webdriver %}
+                    {% if watch_uses_webdriver %}
                     <div class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_screenshot) }}
                         <span class="pure-form-message-inline">
@@ -264,13 +278,39 @@ Math: {{ 1 + 1 }}") }}
                         </div>
                         {% endif %}
                         <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
-
                         {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
                     </div>
                 </fieldset>
             </div>
 
             {% if watch['processor'] == 'text_json_diff' %}
+
+            <div class="tab-pane-inner" id="conditions">
+                    <script>
+                        const verify_condition_rule_url="{{url_for('conditions.verify_condition_single_rule', watch_uuid=uuid)}}";
+                    </script>
+                <style>
+                    .verifyRuleRow {
+                        background-color: #4caf50;
+                        color: white;
+                        border: none;
+                        cursor: pointer;
+                        font-weight: bold;
+                    }
+                    .verifyRuleRow:hover {
+                        background-color: #45a049;
+                    }
+                </style>
+                <div class="pure-control-group">
+                    {{ render_field(form.conditions_match_logic) }}
+                    {{ render_fieldlist_of_formfields_as_table(form.conditions) }}
+                    <div class="pure-form-message-inline">
+                        <br>
+                        Use the verify (✓) button to test if a condition passes against the current snapshot.<br><br>
+                        Did you know that <strong>conditions</strong> can be extended with your own custom plugin? tutorials coming soon!<br>
+                    </div>
+                </div>
+            </div>
             <div class="tab-pane-inner" id="filters-and-triggers">
                 <span id="activate-text-preview" class="pure-button pure-button-primary button-xsmall">Activate preview</span>
               <div>
@@ -298,7 +338,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                           <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
                         {% endif %}
                         <span class="pure-form-message-inline">One CSS, xPath, JSON Path/JQ selector per line, <i>any</i> rules that matches will be used.<br>
-<p><div data-target="#advanced-help-selectors" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</div><br></p>
+                        <span data-target="#advanced-help-selectors" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</span><br>
                     <ul id="advanced-help-selectors" style="display: none;">
                         <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
                         <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
@@ -471,7 +511,7 @@ keyword") }}
 
                 <fieldset>
                     <div class="pure-control-group">
-                        {% if visualselector_enabled %}
+                        {% if playwright_enabled and watch_uses_webdriver %}
                             <span class="pure-form-message-inline" id="visual-selector-heading">
                                 The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection. It automatically fills-in the filters in the "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab. Use <strong>Shift+Click</strong> to select multiple items.
                             </span>
@@ -489,11 +529,12 @@ keyword") }}
                             </div>
                             <div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
                         {% else %}
-                            <span class="pure-form-message-inline">
+                            {% if not watch_uses_webdriver %}
-                                <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
+                                {{ only_webdriver_type_watches_warning() }}
-                                <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
+                            {% endif %}
-                                <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
+                            {% if not playwright_enabled %}
-                            </span>
+                                {{ playwright_warning() }}
+                            {% endif %}
                         {% endif %}
                     </div>
                 </fieldset>

changedetectionio/templates/settings.html

@@ -214,7 +214,7 @@ nav
                         <a id="chrome-extension-link"
                            title="Try our new Chrome Extension!"
                            href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                            <img src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}" alt="Chrome">
+                            <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}" alt="Chrome">
                             Chrome Webstore
                         </a>
                     </p>
@@ -280,9 +280,7 @@ nav
                         
                     </div>
                 </div>
-                <p>
+
-                    Your proxy provider may need to whitelist our IP of <code>204.15.192.195</code>
-                </p>
                <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
 
                 <div class="pure-control-group" id="extra-proxies-setting">

changedetectionio/templates/svgs/search-icon.svg

@@ -1 +1 @@
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>
+<svg version="1.1" id="search" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>

changedetectionio/templates/watch-overview.html

@@ -108,7 +108,8 @@
                     {% else %}
                     <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
                     {% endif %}
-                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
+                    {% set mute_label = 'UnMute notification' if watch.notification_muted else 'Mute notification' %}
+                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="{{ mute_label }}" title="{{ mute_label }}" class="icon icon-mute" ></a>
                 </td>
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
@@ -118,7 +119,7 @@
                          or (  watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
                          or "extra_browser_" in watch.get_fetch_backend
                     %}
-                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" title="Using a Chrome browser" >
+                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" alt="Using a Chrome browser" title="Using a Chrome browser" >
                     {% endif %}
 
                     {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" >{% endif %}

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -34,7 +34,7 @@ def test_execute_custom_js(client, live_server, measure_memory_usage):
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
 
     assert b"This text should be removed" not in res.data

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -48,7 +48,7 @@ def test_noproxy_option(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.get(
         url_for("edit_page", uuid=uuid, unpause_on_save=1))
     assert b'No proxy' in res.data

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -81,7 +81,7 @@ def test_socks5(client, live_server, measure_memory_usage):
     assert "Awesome, you made it".encode('utf-8') in res.data
 
     # PROXY CHECKER WIDGET CHECK - this needs more checking
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     res = client.get(
         url_for("check_proxies.start_check", uuid=uuid),

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
+
 import os.path
-import time
+
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output
-from changedetectionio import html_tools
 
 
 def set_original(excluding=None, add_line=None):
@@ -165,6 +165,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
+
     assert b'unviewed' in res.data
 
     # Takes a moment for apprise to fire

changedetectionio/tests/test_api.py

@@ -44,7 +44,6 @@ def set_modified_response():
 
     return None
 
-
 def is_valid_uuid(val):
     try:
         uuid.UUID(str(val))
@@ -56,8 +55,9 @@ def is_valid_uuid(val):
 def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
+
 def test_api_simple(client, live_server, measure_memory_usage):
-    #live_server_setup(live_server)
+#    live_server_setup(live_server)
 
     api_key = extract_api_key_from_UI(client)
 
@@ -129,6 +129,9 @@ def test_api_simple(client, live_server, measure_memory_usage):
     assert after_recheck_info['last_checked'] != before_recheck_info['last_checked']
     assert after_recheck_info['last_changed'] != 0
 
+    # #2877 When run in a slow fetcher like playwright etc
+    assert after_recheck_info['last_changed'] ==  after_recheck_info['last_checked']
+
     # Check history index list
     res = client.get(
         url_for("watchhistory", uuid=watch_uuid),

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -99,7 +99,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
     assert b'ldjson-price-track-offer' in res.data
 
     # Accept it
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     #time.sleep(1)
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
     client.get(url_for("form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_backend.py

@@ -2,7 +2,6 @@
 
 import time
 from flask import url_for
-from urllib.request import urlopen
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
     extract_UUID_from_client
 
@@ -69,7 +68,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     wait_for_all_checks(client)
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     # Check the 'get latest snapshot works'
     res = client.get(url_for("watch_get_latest_html", uuid=uuid))

changedetectionio/tests/test_conditions.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+
+from flask import url_for
+from .util import live_server_setup, wait_for_all_checks
+
+def set_original_response(number="50"):
+    test_return_data = f"""<html>
+       <body>
+     <h1>Test Page for Conditions</h1>
+     <p>This page contains a number that will be tested with conditions.</p>
+     <div class="number-container">Current value: {number}</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+def set_number_in_range_response(number="75"):
+    test_return_data = f"""<html>
+       <body>
+     <h1>Test Page for Conditions</h1>
+     <p>This page contains a number that will be tested with conditions.</p>
+     <div class="number-container">Current value: {number}</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+def set_number_out_of_range_response(number="150"):
+    test_return_data = f"""<html>
+       <body>
+     <h1>Test Page for Conditions</h1>
+     <p>This page contains a number that will be tested with conditions.</p>
+     <div class="number-container">Current value: {number}</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_conditions_with_text_and_number(client, live_server):
+    """Test that both text and number conditions work together with AND logic."""
+    
+    set_original_response("50")
+    live_server_setup(live_server)
+
+    test_url = url_for('test_endpoint', _external=True)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    # Configure the watch with two conditions connected with AND:
+    # 1. The page filtered text must contain "5" (first digit of value)
+    # 2. The extracted number should be >= 20 and <= 100
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "url": test_url,
+            "fetch_backend": "html_requests",
+            "include_filters": ".number-container",
+            "title": "Number AND Text Condition Test",
+            "conditions_match_logic": "ALL",  # ALL = AND logic
+            "conditions-0-operator": "in",
+            "conditions-0-field": "page_filtered_text",
+            "conditions-0-value": "5",
+
+            "conditions-1-operator": ">=",
+            "conditions-1-field": "extracted_number",
+            "conditions-1-value": "20",
+
+            "conditions-2-operator": "<=",
+            "conditions-2-field": "extracted_number",
+            "conditions-2-value": "100",
+
+            # So that 'operations' from pluggy discovery are tested
+            "conditions-3-operator": "length_min",
+            "conditions-3-field": "page_filtered_text",
+            "conditions-3-value": "1",
+
+            # So that 'operations' from pluggy discovery are tested
+            "conditions-4-operator": "length_max",
+            "conditions-4-field": "page_filtered_text",
+            "conditions-4-value": "100",
+
+            # So that 'operations' from pluggy discovery are tested
+            "conditions-5-operator": "contains_regex",
+            "conditions-5-field": "page_filtered_text",
+            "conditions-5-value": "\d",
+        },
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    wait_for_all_checks(client)
+    client.get(url_for("mark_all_viewed"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    # Case 1
+    set_number_in_range_response("70.5")
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    # 75 is > 20 and < 100 and contains "5"
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+
+    # Case 2: Change with one condition violated
+    # Number out of range (150) but contains '5'
+    client.get(url_for("mark_all_viewed"), follow_redirects=True)
+    set_number_out_of_range_response("150.5")
+
+
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    # Should NOT be marked as having changes since not all conditions are met
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

changedetectionio/tests/test_encoding.py

@@ -40,7 +40,7 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage):
 
 
     # Content type recording worked
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['content-type'] == "text/html"
 
     res = client.get(

changedetectionio/tests/test_filter_failure_notification.py

@@ -51,7 +51,7 @@ def run_filter_test(client, live_server, content_filter):
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 0, "No filter = No filter failure"
 

changedetectionio/tests/test_group.py

@@ -288,7 +288,7 @@ def test_clone_tag_on_import(client, live_server, measure_memory_usage):
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
-    watch_uuid = extract_UUID_from_client(client)
+    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
 
     assert b'Cloned' in res.data
@@ -315,7 +315,7 @@ def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usa
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
-    watch_uuid = extract_UUID_from_client(client)
+    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
 
     assert b'Cloned' in res.data

changedetectionio/tests/test_ignore.py

@@ -36,7 +36,7 @@ def test_ignore(client, live_server, measure_memory_usage):
 
     # Give the thread time to pick it up
     wait_for_all_checks(client)
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     # use the highlighter endpoint
     res = client.post(
         url_for("highlight_submit_ignore_url", uuid=uuid),

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -514,3 +514,15 @@ def test_check_jq_ext_filter(client, live_server, measure_memory_usage):
 def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage):
     if jq_support:
         check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
+
+def test_jsonpath_BOM_utf8(client, live_server, measure_memory_usage):
+    from .. import html_tools
+
+    # JSON string with BOM and correct double-quoted keys
+    json_str = '\ufeff{"name": "José", "emoji": "😊", "language": "中文", "greeting": "Привет"}'
+
+    # See that we can find the second <script> one, which is not broken, and matches our filter
+    text = html_tools.extract_json_as_string(json_str, "json:$.name")
+    assert text == '"José"'
+
+    

changedetectionio/tests/test_live_preview.py

@@ -29,7 +29,7 @@ def test_content_filter_live_preview(client, live_server, measure_memory_usage):
         data={"url": test_url, "tags": ''},
         follow_redirects=True
     )
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.post(
         url_for("edit_page", uuid=uuid),
         data={

changedetectionio/tests/test_nonrenderable_pages.py

@@ -48,7 +48,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     #####################
     client.post(
         url_for("settings_page"),
-        data={"application-empty_pages_are_a_change": "",
+        data={"application-empty_pages_are_a_change": "", # default, OFF, they are NOT a change
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
@@ -66,6 +66,14 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
+
+    assert watch.last_changed == 0
+    assert watch['last_checked'] != 0
+
+
+
 
     # ok now do the opposite
 
@@ -92,6 +100,10 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # A totally zero byte (#2528) response should also not trigger an error
     set_zero_byte_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
+    # 2877
+    assert watch.last_changed == watch['last_checked']
+
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data # A change should have registered because empty_pages_are_a_change is ON

changedetectionio/tests/test_notification.py

@@ -6,7 +6,7 @@ from flask import url_for
 from loguru import logger
 
 from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \
-    set_longer_modified_response
+    set_longer_modified_response, get_index
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -29,7 +29,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
     # Re 360 - new install should have defaults set
     res = client.get(url_for("settings_page"))
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')+"?status_code=204"
 
     assert default_notification_body.encode() in res.data
     assert default_notification_title.encode() in res.data
@@ -76,7 +76,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
     testimage_png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='
 
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     datastore = 'test-datastore'
     with open(os.path.join(datastore, str(uuid), 'last-screenshot.png'), 'wb') as f:
         f.write(base64.b64decode(testimage_png))
@@ -135,7 +135,14 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
     time.sleep(3)
+
+    # Check no errors were recorded
+    res = client.get(url_for("index"))
+    assert b'notification-error' not in res.data
+
+
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
         notification_submission = f.read()
@@ -284,7 +291,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
 
     res = client.post(
         url_for("settings_page"),
@@ -319,6 +326,11 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
 
     time.sleep(2) # plus extra delay for notifications to fire
 
+
+    # Check no errors were recorded, because we asked for 204 which is slightly uncommon but is still OK
+    res = get_index(client)
+    assert b'notification-error' not in res.data
+
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
         j = json.loads(x)
@@ -360,7 +372,10 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     #live_server_setup(live_server)
     set_original_response()
     if os.path.isfile("test-datastore/notification.txt"):
-        os.unlink("test-datastore/notification.txt")
+        os.unlink("test-datastore/notification.txt") \
+
+    # 1995 UTF-8 content should be encoded
+    test_body = 'change detection is cool 网站监测 内容更新了'
 
     # otherwise other settings would have already existed from previous tests in this file
     res = client.post(
@@ -368,8 +383,7 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
         data={
             "application-fetch_backend": "html_requests",
             "application-minutes_between_check": 180,
-            #1995 UTF-8 content should be encoded
+            "application-notification_body": test_body,
-            "application-notification_body": 'change detection is cool 网站监测 内容更新了',
             "application-notification_format": default_notification_format,
             "application-notification_urls": "",
             "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
@@ -399,12 +413,10 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert res.status_code != 400
     assert res.status_code != 500
 
-    # Give apprise time to fire
-    time.sleep(4)
 
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
-        assert 'change detection is cool 网站监测 内容更新了' in x
+        assert test_body in x
 
     os.unlink("test-datastore/notification.txt")
 

changedetectionio/tests/test_processor_registry.py

@@ -0,0 +1,59 @@
+import pytest
+from changedetectionio.processors.processor_registry import get_processor_class, get_all_processors
+
+
+def test_get_all_processors():
+    """Test that get_all_processors returns a list of processor tuples"""
+    processors = get_all_processors()
+    assert isinstance(processors, list)
+    assert len(processors) > 0
+    
+    # Each item should be a tuple of (name, description)
+    for processor in processors:
+        assert isinstance(processor, tuple)
+        assert len(processor) == 2
+        assert isinstance(processor[0], str)
+        assert isinstance(processor[1], str)
+        
+    # Check that our WHOIS processor is included
+    whois_processor = next((p for p in processors if p[0] == "whois"), None)
+    assert whois_processor is not None
+    assert whois_processor[1] == "WHOIS Domain Information Changes"
+
+
+def test_get_processor_class():
+    """Test that get_processor_class returns the right class"""
+    # Get the WHOIS processor class
+    processor_class = get_processor_class("whois")
+    assert processor_class is not None
+    
+    # It should have perform_site_check method
+    assert hasattr(processor_class, 'perform_site_check')
+    
+    # Check for non-existent processor
+    non_existent = get_processor_class("non_existent_processor")
+    assert non_existent is None
+
+
+def test_get_processor_site_check():
+    """Test that get_processor_site_check returns a processor instance"""
+    from unittest.mock import MagicMock
+    from changedetectionio.processors.processor_registry import get_processor_site_check
+    
+    # Get a WHOIS processor instance
+    mock_datastore = MagicMock()
+    watch_uuid = "test-uuid"
+    processor = get_processor_site_check("whois", mock_datastore, watch_uuid)
+    
+    # It should be a processor instance
+    assert processor is not None
+    
+    # It should have the run_changedetection method
+    assert hasattr(processor, 'run_changedetection')
+    
+    # It should have the call_browser method
+    assert hasattr(processor, 'call_browser')
+    
+    # Check for non-existent processor
+    non_existent = get_processor_site_check("non_existent_processor", mock_datastore, watch_uuid)
+    assert non_existent is None

changedetectionio/tests/test_request.py

@@ -373,13 +373,14 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     with open('test-datastore/headers-testtag.txt', 'w') as f:
-        f.write("tag-header: test")
+        f.write("tag-header: test\r\nurl-header: http://example.com")
 
     with open('test-datastore/headers.txt', 'w') as f:
-        f.write("global-header: nice\r\nnext-global-header: nice")
+        f.write("global-header: nice\r\nnext-global-header: nice\r\nurl-header-global: http://example.com/global")
 
-    with open('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt', 'w') as f:
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-        f.write("watch-header: nice")
+    with open(f'test-datastore/{uuid}/headers.txt', 'w') as f:
+        f.write("watch-header: nice\r\nurl-header-watch: http://example.com/watch")
 
     wait_for_all_checks(client)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -410,6 +411,9 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     assert b"Xxx:ooo" in res.data
     assert b"Watch-Header:nice" in res.data
     assert b"Tag-Header:test" in res.data
+    assert b"Url-Header:http://example.com" in res.data
+    assert b"Url-Header-Global:http://example.com/global" in res.data
+    assert b"Url-Header-Watch:http://example.com/watch" in res.data
 
     # Check the custom UA from system settings page made it through
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):

changedetectionio/tests/test_restock_itemprop.py

@@ -189,6 +189,17 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
 
     client.get(url_for("mark_all_viewed"))
 
+
+    # 2715 - Price detection (once it crosses the "lower" threshold) again with a lower price - should trigger again!
+    set_original_response(props_markup=instock_props[0], price='820.45')
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    assert b'1 watches queued for rechecking.' in res.data
+    wait_for_all_checks(client)
+    res = client.get(url_for("index"))
+    assert b'820.45' in res.data
+    assert b'unviewed' in res.data
+    client.get(url_for("mark_all_viewed"))
+
     # price changed to something MORE than max (1100.10), SHOULD be a change
     set_original_response(props_markup=instock_props[0], price='1890.45')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -203,7 +214,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
 
 
 def test_restock_itemprop_minmax(client, live_server):
-#    live_server_setup(live_server)
+    #live_server_setup(live_server)
     extras = {
         "restock_settings-follow_price_changes": "y",
         "restock_settings-price_change_min": 900.0,
@@ -369,7 +380,7 @@ def test_change_with_notification_values(client, live_server):
 
     ## Now test the "SEND TEST NOTIFICATION" is working
     os.unlink("test-datastore/notification.txt")
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.post(url_for("ajax_callback_send_notification_test", watch_uuid=uuid), data={}, follow_redirects=True)
     time.sleep(5)
     assert os.path.isfile("test-datastore/notification.txt"), "Notification received"

changedetectionio/tests/test_rss.py

@@ -132,7 +132,7 @@ def test_rss_xpath_filtering(client, live_server, measure_memory_usage):
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
         data={

changedetectionio/tests/test_scheduler.py

@@ -39,7 +39,7 @@ def test_check_basic_scheduler_functionality(client, live_server, measure_memory
 
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
 
@@ -104,7 +104,7 @@ def test_check_basic_global_scheduler_functionality(client, live_server, measure
 
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
 

changedetectionio/tests/test_whois_processor.py

@@ -0,0 +1,182 @@
+import pytest
+from unittest.mock import MagicMock, patch
+from changedetectionio.processors.whois_plugin import WhoisProcessor
+
+
+class MockWatch:
+    def __init__(self, url, previous_md5=None, include_filters=None, ignore_text=None):
+        self.url = url
+        self._previous_md5 = previous_md5
+        self._include_filters = include_filters or []
+        self._ignore_text = ignore_text or []
+        self.history = {}
+    
+    def get(self, key, default=None):
+        if key == 'previous_md5':
+            return self._previous_md5
+        elif key == 'include_filters':
+            return self._include_filters
+        elif key == 'ignore_text':
+            return self._ignore_text
+        elif key == 'url':
+            return self.url
+        return default
+    
+    def has_special_diff_filter_options_set(self):
+        return False
+
+
+@patch('whois.whois')
+@patch('changedetectionio.processors.difference_detection_processor.__init__')
+@patch('changedetectionio.processors.text_json_diff.processor.perform_site_check.run_changedetection')
+def test_whois_processor_basic_functionality(mock_super_run, mock_base_init, mock_whois):
+    """Test the basic functionality of the WhoisProcessor"""
+    # Mock the base class init so we don't need to set up the full watch structure
+    mock_base_init.return_value = None
+    
+    # Mock super().run_changedetection to return a simple result
+    mock_super_run.return_value = (False, {'previous_md5': 'some-md5'}, b'Some filtered text')
+    
+    # Mock the whois response
+    mock_whois_result = MagicMock()
+    mock_whois_result.text = "Domain Name: example.com\nRegistrar: Example Registrar\nCreation Date: 2020-01-01\n"
+    mock_whois.return_value = mock_whois_result
+    
+    # Create mock datastore
+    mock_datastore = MagicMock()
+    mock_datastore.proxy_list = None  # No proxies
+    mock_datastore.get_preferred_proxy_for_watch.return_value = None
+    mock_datastore.data = {
+        'settings': {
+            'application': {
+                'allow_file_uri': False
+            }
+        }
+    }
+    
+    # Create a processor instance and setup minimal required attributes
+    processor = WhoisProcessor(datastore=mock_datastore, watch_uuid='test-uuid')
+    
+    # Create a minimal watch object
+    watch = MockWatch(url="https://example.com")
+    
+    # Simulate link access in the watch
+    processor.watch = MagicMock()
+    processor.watch.link = "https://example.com"
+    processor.watch.get.return_value = "uuid-123"
+    
+    # Run the processor's run_changedetection method by first using call_browser
+    processor.call_browser()
+    
+    # Check that the fetcher was set up correctly
+    assert processor.fetcher is not None
+    assert hasattr(processor.fetcher, 'content')
+    assert hasattr(processor.fetcher, 'headers')
+    assert hasattr(processor.fetcher, 'status_code')
+    
+    # Verify that whois was called with the right domain
+    assert mock_whois.called
+    assert mock_whois.call_args[0][0] == 'example.com'
+    
+    # Now run the processor
+    result = processor.run_changedetection(watch)
+    
+    # Check that the parent run_changedetection was called
+    assert mock_super_run.called
+
+
+@patch('whois.whois')
+@patch('changedetectionio.processors.difference_detection_processor.__init__')
+def test_whois_processor_call_browser_with_proxy(mock_base_init, mock_whois):
+    """Test the call_browser method with proxy configuration"""
+    # Mock the base class init
+    mock_base_init.return_value = None
+    
+    # Mock the whois response
+    mock_whois_result = MagicMock()
+    mock_whois_result.text = "Domain Name: example.com\nRegistrar: Example Registrar\nCreation Date: 2020-01-01\n"
+    mock_whois.return_value = mock_whois_result
+    
+    # Create mock datastore
+    mock_datastore = MagicMock()
+    mock_proxy = {
+        'test-proxy': {
+            'url': 'http://proxy.example.com:8080',
+            'label': 'Test Proxy'
+        }
+    }
+    mock_datastore.proxy_list = mock_proxy
+    mock_datastore.get_preferred_proxy_for_watch.return_value = 'test-proxy'
+    mock_datastore.data = {
+        'settings': {
+            'application': {
+                'allow_file_uri': False
+            }
+        }
+    }
+    
+    # Create a processor instance with our mock datastore
+    processor = WhoisProcessor(datastore=mock_datastore, watch_uuid='test-uuid')
+    
+    # Set up watch
+    processor.watch = MagicMock()
+    processor.watch.link = "https://example.com"
+    processor.watch.get.return_value = "uuid-123"
+    
+    # Call the method with a proxy
+    processor.call_browser()
+    
+    # Verify whois was called
+    assert mock_whois.called
+    assert mock_whois.call_args[0][0] == 'example.com'
+    
+    # Check that the fetcher was set up correctly
+    assert processor.fetcher is not None
+    assert processor.fetcher.content is not None
+
+
+@patch('changedetectionio.processors.difference_detection_processor.__init__')
+def test_whois_processor_perform_site_check(mock_base_init):
+    """Test the WhoisProcessor.perform_site_check static method"""
+    mock_base_init.return_value = None
+    
+    # Test the static method
+    with patch.object(WhoisProcessor, '__init__', return_value=None) as mock_init:
+        datastore = MagicMock()
+        watch_uuid = "test-uuid"
+        
+        # Call the static method
+        processor = WhoisProcessor.perform_site_check(datastore=datastore, watch_uuid=watch_uuid)
+        
+        # Check that constructor was called with expected args
+        mock_init.assert_called_once_with(datastore=datastore, watch_uuid=watch_uuid)
+        
+        # Check it returns the right type
+        assert isinstance(processor, WhoisProcessor)
+
+
+def test_get_display_link():
+    """Test the get_display_link hook implementation"""
+    from changedetectionio.processors.whois_plugin import get_display_link
+    
+    # Test with a regular URL
+    url = "https://example.com/some/path?param=value"
+    processor_name = "whois"
+    link = get_display_link(url=url, processor_name=processor_name)
+    assert link == "WHOIS - example.com"
+    
+    # Test with a subdomain
+    url = "https://subdomain.example.com/"
+    link = get_display_link(url=url, processor_name=processor_name)
+    assert link == "WHOIS - subdomain.example.com"
+    
+    # Test with www prefix (should be removed)
+    url = "https://www.example.com/"
+    link = get_display_link(url=url, processor_name=processor_name)
+    assert link == "WHOIS - example.com"
+    
+    # Test with a different processor (should return None)
+    url = "https://example.com/"
+    processor_name = "text_json_diff"
+    link = get_display_link(url=url, processor_name=processor_name)
+    assert link is None

changedetectionio/tests/unit/test_conditions.py

@@ -0,0 +1,82 @@
+from changedetectionio.conditions import execute_ruleset_against_all_plugins
+from changedetectionio.store import ChangeDetectionStore
+import shutil
+import tempfile
+import time
+import unittest
+import uuid
+
+
+class TestTriggerConditions(unittest.TestCase):
+    def setUp(self):
+
+        # Create a temporary directory for the test datastore
+        self.test_datastore_path = tempfile.mkdtemp()
+
+        # Initialize ChangeDetectionStore with our test path and no default watches
+        self.store = ChangeDetectionStore(
+            datastore_path=self.test_datastore_path,
+            include_default_watches=False
+        )
+
+        # Add a test watch
+        watch_url = "https://example.com"
+        self.watch_uuid = self.store.add_watch(url=watch_url)
+
+    def tearDown(self):
+      # Clean up the test datastore
+      self.store.stop_thread = True
+      time.sleep(0.5)  # Give thread time to stop
+      shutil.rmtree(self.test_datastore_path)
+
+    def test_conditions_execution_pass(self):
+        # Get the watch object
+        watch = self.store.data['watching'][self.watch_uuid]
+
+        # Create and save a snapshot
+        first_content = "I saw 100 people at a rock show"
+        timestamp1 = int(time.time())
+        snapshot_id1 = str(uuid.uuid4())
+        watch.save_history_text(contents=first_content,
+                                timestamp=timestamp1,
+                                snapshot_id=snapshot_id1)
+
+        # Add another snapshot
+        second_content = "I saw 200 people at a rock show"
+        timestamp2 = int(time.time()) + 60
+        snapshot_id2 = str(uuid.uuid4())
+        watch.save_history_text(contents=second_content,
+                                timestamp=timestamp2,
+                                snapshot_id=snapshot_id2)
+
+        # Verify both snapshots are stored
+        history = watch.history
+        self.assertEqual(len(history), 2)
+
+        # Retrieve and check snapshots
+        #snapshot1 = watch.get_history_snapshot(str(timestamp1))
+        #snapshot2 = watch.get_history_snapshot(str(timestamp2))
+
+        self.store.data['watching'][self.watch_uuid].update(
+            {
+                "conditions_match_logic": "ALL",
+                "conditions": [
+                    {"operator": ">=", "field": "extracted_number", "value": "10"},
+                    {"operator": "<=", "field": "extracted_number", "value": "5000"},
+                    {"operator": "in", "field": "page_text", "value": "rock"},
+                    #{"operator": "starts_with", "field": "page_text", "value": "I saw"},
+                ]
+            }
+        )
+
+        # ephemeral_data - some data that could exist before the watch saved a new version
+        result = execute_ruleset_against_all_plugins(current_watch_uuid=self.watch_uuid,
+                                                     application_datastruct=self.store.data,
+                                                     ephemeral_data={'text': "I saw 500 people at a rock show"})
+
+        # @todo - now we can test that 'Extract number' increased more than X since last time
+        self.assertTrue(result)
+
+
+if __name__ == '__main__':
+    unittest.main()

changedetectionio/tests/unit/test_semver.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+# run from dir above changedetectionio/ dir
+# python3 -m unittest changedetectionio.tests.unit.test_semver
+
+import re
+import unittest
+
+
+# The SEMVER regex
+SEMVER_REGEX = r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
+
+# Compile the regex
+semver_pattern = re.compile(SEMVER_REGEX)
+
+class TestSemver(unittest.TestCase):
+    def test_valid_versions(self):
+        """Test valid semantic version strings"""
+        valid_versions = [
+            "1.0.0",
+            "0.1.0",
+            "0.0.1",
+            "1.0.0-alpha",
+            "1.0.0-alpha.1",
+            "1.0.0-0.3.7",
+            "1.0.0-x.7.z.92",
+            "1.0.0-alpha+001",
+            "1.0.0+20130313144700",
+            "1.0.0-beta+exp.sha.5114f85"
+        ]
+        for version in valid_versions:
+            with self.subTest(version=version):
+                self.assertIsNotNone(semver_pattern.match(version), f"Version {version} should be valid")
+
+    def test_invalid_versions(self):
+        """Test invalid semantic version strings"""
+        invalid_versions = [
+            "0.48.06",
+            "1.0",
+            "1.0.0-",
+# Seems to pass the semver.org regex?
+#            "1.0.0-alpha-",
+            "1.0.0+",
+            "1.0.0-alpha+",
+            "1.0.0-",
+            "01.0.0",
+            "1.01.0",
+            "1.0.01",
+            ".1.0.0",
+            "1..0.0"
+        ]
+        for version in invalid_versions:
+            with self.subTest(version=version):
+                res = semver_pattern.match(version)
+                self.assertIsNone(res, f"Version '{version}' should be invalid")
+
+    def test_our_version(self):
+        from changedetectionio import get_version
+        our_version = get_version()
+        self.assertIsNotNone(semver_pattern.match(our_version), f"Our version '{our_version}' should be a valid SEMVER string")
+
+
+if __name__ == '__main__':
+    unittest.main()

changedetectionio/tests/unit/test_watch_model.py

@@ -61,5 +61,22 @@ class TestDiffBuilder(unittest.TestCase):
         p = watch.get_from_version_based_on_last_viewed
         assert p == "100", "Correct with only one history snapshot"
 
+    def test_watch_link_property_with_processor(self):
+        """Test the link property with a processor that customizes the link"""
+        from unittest.mock import patch
+        
+        watch = Watch.model(datastore_path='/tmp', default={})
+        watch['url'] = 'https://example.com'
+        watch['processor'] = 'whois'
+        
+        # Mock the processor registry's get_display_link function
+        with patch('changedetectionio.processors.processor_registry.get_display_link') as mock_get_display_link:
+            mock_get_display_link.return_value = "WHOIS - example.com"
+            
+            # The link property should use the customized link from the processor
+            assert watch.link == "WHOIS - example.com"
+            mock_get_display_link.assert_called_once_with(url='https://example.com', processor_name='whois')
+
+
 if __name__ == '__main__':
     unittest.main()

changedetectionio/tests/util.py

@@ -76,6 +76,14 @@ def set_more_modified_response():
     return None
 
 
+def set_empty_text_response():
+    test_return_data = """<html><body></body></html>"""
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
 def wait_for_notification_endpoint_output():
     '''Apprise can take a few seconds to fire'''
     #@todo - could check the apprise object directly instead of looking for this file
@@ -215,9 +223,10 @@ def live_server_setup(live_server):
     def test_method():
         return request.method
 
-    # Where we POST to as a notification
+    # Where we POST to as a notification, also use a space here to test URL escaping is OK across all tests that use this. ( #2868 )
-    @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET'])
+    @live_server.app.route('/test_notification endpoint', methods=['POST', 'GET'])
     def test_notification_endpoint():
+
         with open("test-datastore/notification.txt", "wb") as f:
             # Debug method, dump all POST to file also, used to prove #65
             data = request.stream.read()
@@ -235,8 +244,11 @@ def live_server_setup(live_server):
                 f.write(request.content_type)
 
         print("\n>> Test notification endpoint was hit.\n", data)
-        return "Text was set"
 
+        content = "Text was set"
+        status_code = request.args.get('status_code',200)
+        resp = make_response(content, status_code)
+        return resp
 
     # Just return the verb in the request
     @live_server.app.route('/test-basicauth', methods=['GET'])
@@ -273,15 +285,43 @@ def live_server_setup(live_server):
             <p id="remove">This text should be removed</p>
               <form onsubmit="event.preventDefault();">
             <!-- obfuscated text so that we dont accidentally get a false positive due to conversion of the source :) --->
-                <button name="test-button" onclick="getElementById('remove').remove();getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=')">Click here</button>
+                <button name="test-button" onclick="
-                <div id=some-content></div>
+                getElementById('remove').remove();
+                getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=');
+                getElementById('reflect-text').innerHTML = getElementById('test-input-text').value;
+                ">Click here</button>
+                
+                <div id="some-content"></div>
+                
                 <pre>
                 {header_text.lower()}
                 </pre>
-              </body>
+                
+                <br>
+                <!-- used for testing that the jinja2 compiled here --->
+                <input type="text" value="" id="test-input-text" /><br>
+                <div id="reflect-text">Waiting to reflect text from #test-input-text here</div>
+              </form>
+                
+           </body>
          </html>""", 200)
         resp.headers['Content-Type'] = 'text/html'
         return resp
 
     live_server.start()
 
+def get_index(client):
+    import inspect
+    # Get the caller's frame (parent function)
+    frame = inspect.currentframe()
+    caller_frame = frame.f_back  # Go back to the caller's frame
+    caller_name = caller_frame.f_code.co_name
+    caller_line = caller_frame.f_lineno
+
+    print(f"Called by: {caller_name}, Line: {caller_line}")
+
+    res = client.get(url_for("index"))
+    with open(f"test-datastore/index-{caller_name}-{caller_line}.html", 'wb') as f:
+        f.write(res.data)
+
+    return res

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -2,14 +2,16 @@
 
 import os
 from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
+from ..util import live_server_setup, wait_for_all_checks, get_index
 
-def test_setup(client, live_server, measure_memory_usage):
+def test_setup(client, live_server):
     live_server_setup(live_server)
 
 
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
 def test_visual_selector_content_ready(client, live_server, measure_memory_usage):
+    live_server.stop()
+    live_server.start()
 
     import os
     import json
@@ -27,7 +29,7 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
         data={
@@ -87,7 +89,9 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
 
 def test_basic_browserstep(client, live_server, measure_memory_usage):
 
-    #live_server_setup(live_server)
+    live_server.stop()
+    live_server.start()
+
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
 
     test_url = url_for('test_interactive_html_endpoint', _external=True)
@@ -108,9 +112,13 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
             "url": test_url,
             "tags": "",
             'fetch_backend': "html_webdriver",
-            'browser_steps-0-operation': 'Click element',
+            'browser_steps-0-operation': 'Enter text in field',
-            'browser_steps-0-selector': 'button[name=test-button]',
+            'browser_steps-0-selector': '#test-input-text',
-            'browser_steps-0-optional_value': '',
+            # Should get set to the actual text (jinja2 rendered)
+            'browser_steps-0-optional_value': "Hello-Jinja2-{% now  'Europe/Berlin', '%Y-%m-%d' %}",
+            'browser_steps-1-operation': 'Click element',
+            'browser_steps-1-selector': 'button[name=test-button]',
+            'browser_steps-1-optional_value': '',
             # For now, cookies doesnt work in headers because it must be a full cookiejar object
             'headers': "testheader: yes\buser-agent: MyCustomAgent",
         },
@@ -119,7 +127,7 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
 
     assert b"This text should be removed" not in res.data
@@ -132,13 +140,32 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
     assert b"This text should be removed" not in res.data
     assert b"I smell JavaScript because the button was pressed" in res.data
 
+    assert b'Hello-Jinja2-20' in res.data
+
     assert b"testheader: yes" in res.data
     assert b"user-agent: mycustomagent" in res.data
+    live_server.stop()
+
+def test_non_200_errors_report_browsersteps(client, live_server):
+
+    live_server.stop()
+    live_server.start()
 
     four_o_four_url =  url_for('test_endpoint', status_code=404, _external=True)
     four_o_four_url = four_o_four_url.replace('localhost.localdomain', 'cdio')
     four_o_four_url = four_o_four_url.replace('localhost', 'cdio')
 
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": four_o_four_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added in Paused state, saving will unpause" in res.data
+    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
+
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+
     # now test for 404 errors
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
@@ -153,12 +180,14 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     assert b"unpaused" in res.data
+
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = get_index(client)
+
     assert b'Error - 404' in res.data
 
     client.get(
         url_for("form_delete", uuid="all"),
         follow_redirects=True
-    )
+    )

changedetectionio/update_worker.py

@@ -243,7 +243,6 @@ class update_worker(threading.Thread):
                 os.unlink(full_path)
 
     def run(self):
-        now = time.time()
         
         while not self.app.config.exit.is_set():
             update_handler = None
@@ -254,6 +253,7 @@ class update_worker(threading.Thread):
                 pass
 
             else:
+                fetch_start_time = time.time()
                 uuid = queued_item_data.item.get('uuid')
                 self.current_uuid = uuid
                 if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
@@ -268,26 +268,48 @@ class update_worker(threading.Thread):
                     watch = self.datastore.data['watching'].get(uuid)
 
                     logger.info(f"Processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
-                    now = time.time()
 
                     try:
                         # Processor is what we are using for detecting the "Change"
-                        processor = watch.get('processor', 'text_json_diff')
+                        processor_name = watch.get('processor', 'text_json_diff')
 
-                        # Init a new 'difference_detection_processor', first look in processors
+                        
-                        processor_module_name = f"changedetectionio.processors.{processor}.processor"
+
+                        # First, try to get the processor from our plugin registry
                         try:
-                            processor_module = importlib.import_module(processor_module_name)
+                            from changedetectionio.processors.processor_registry import get_processor_site_check
-                        except ModuleNotFoundError as e:
+                            update_handler = get_processor_site_check(processor_name, self.datastore, uuid)
-                            print(f"Processor module '{processor}' not found.")
+                            
-                            raise e
+                            if update_handler:
-
+                                # We found the processor in our plugin registry
-                        update_handler = processor_module.perform_site_check(datastore=self.datastore,
+                                logger.info(f"Using processor '{processor_name}' from plugin registry")
-                                                                             watch_uuid=uuid
+                            else:
-                                                                             )
+                                # Fall back to the traditional file-based approach
+                                processor_module_name = f"changedetectionio.processors.{processor_name}.processor"
+                                try:
+                                    processor_module = importlib.import_module(processor_module_name)
+                                    update_handler = processor_module.perform_site_check(datastore=self.datastore,
+                                                                                        watch_uuid=uuid)
+                                except ModuleNotFoundError as e:
+                                    print(f"Processor module '{processor_name}' not found in both plugin registry and file system.")
+                                    raise e
+                        except ImportError as e:
+                            # If processor_registry.py cannot be imported, fall back to the traditional approach
+                            processor_module_name = f"changedetectionio.processors.{processor_name}.processor"
+                            try:
+                                processor_module = importlib.import_module(processor_module_name)
+                                update_handler = processor_module.perform_site_check(datastore=self.datastore,
+                                                                                    watch_uuid=uuid)
+                            except ModuleNotFoundError as e:
+                                print(f"Processor module '{processor_name}' not found.")
+                                raise e
 
                         update_handler.call_browser()
 
+                        # In reality, the actual time of when the change was detected could be a few seconds after this
+                        # For example it should include when the page stopped rendering if using a playwright/chrome type fetch
+                        fetch_start_time = time.time()
+
                         changed_detected, update_obj, contents = update_handler.run_changedetection(watch=watch)
 
                         # Re #342
@@ -512,7 +534,7 @@ class update_worker(threading.Thread):
 
                     if not self.datastore.data['watching'].get(uuid):
                         continue
-                    #
+
                     # Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
                     if process_changedetection_results:
 
@@ -525,8 +547,6 @@ class update_worker(threading.Thread):
                                 except Exception as e:
                                     logger.warning(f"UUID: {uuid} Extract <title> as watch title was enabled, but couldn't find a <title>.")
 
-                        # Now update after running everything
-                        timestamp = round(time.time())
                         try:
                             self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
 
@@ -542,24 +562,28 @@ class update_worker(threading.Thread):
 
                                 # Small hack so that we sleep just enough to allow 1 second  between history snapshots
                                 # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
-
+                                # @also - the keys are one per second at the most (for now)
-                                if watch.newest_history_key and int(timestamp) == int(watch.newest_history_key):
+                                if watch.newest_history_key and int(fetch_start_time) == int(watch.newest_history_key):
                                     logger.warning(
-                                        f"Timestamp {timestamp} already exists, waiting 1 seconds so we have a unique key in history.txt")
+                                        f"Timestamp {fetch_start_time} already exists, waiting 1 seconds so we have a unique key in history.txt")
-                                    timestamp = str(int(timestamp) + 1)
+                                    fetch_start_time += 1
                                     time.sleep(1)
 
                                 watch.save_history_text(contents=contents,
-                                                        timestamp=timestamp,
+                                                        timestamp=int(fetch_start_time),
                                                         snapshot_id=update_obj.get('previous_md5', 'none'))
 
-                                if update_handler.fetcher.content:
+
-                                    watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=timestamp)
+                                empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
+                                if update_handler.fetcher.content or (not update_handler.fetcher.content and empty_pages_are_a_change):
+                                    # attribute .last_changed is then based on this data
+                                    watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=int(fetch_start_time))
 
                                 # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
                                 if watch.history_n >= 2:
                                     logger.info(f"Change detected in UUID {uuid} - {watch['url']}")
                                     if not watch.get('notification_muted'):
+                                        # @todo only run this if notifications exist
                                         self.send_content_changed_notification(watch_uuid=uuid)
 
                         except Exception as e:
@@ -581,15 +605,15 @@ class update_worker(threading.Thread):
                     except Exception as e:
                         pass
 
-                    self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
+                    self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - fetch_start_time, 3),
-                                                                       'last_checked': round(time.time()),
+                                                                       'last_checked': int(fetch_start_time),
                                                                        'check_count': count
                                                                        })
 
 
                 self.current_uuid = None  # Done
                 self.q.task_done()
-                logger.debug(f"Watch {uuid} done in {time.time()-now:.2f}s")
+                logger.debug(f"Watch {uuid} done in {time.time()-fetch_start_time:.2f}s")
 
                 # Give the CPU time to interrupt
                 time.sleep(0.1)

docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3.2'
 services:
     changedetection:
       image: ghcr.io/dgtlmoon/changedetection.io
@@ -82,7 +81,7 @@ services:
 
 
      # Sockpuppetbrowser is basically chrome wrapped in an API for allowing fast fetching of web-pages.
-     # RECOMMENDED FOR FETCHING PAGES WITH CHROME
+     # RECOMMENDED FOR FETCHING PAGES WITH CHROME, be sure to enable the "PLAYWRIGHT_DRIVER_URL" env variable in the main changedetection container
 #    sockpuppetbrowser:
 #        hostname: sockpuppetbrowser
 #        image: dgtlmoon/sockpuppetbrowser:latest

requirements.txt

@@ -35,7 +35,7 @@ dnspython==2.6.1 # related to eventlet fixes
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise==1.9.0
+apprise==1.9.2
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 # use any version other than 2.0.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
@@ -95,5 +95,20 @@ babel
 # Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096
 greenlet >= 3.0.3
 
+# Pinned or it causes problems with flask_expects_json which seems unmaintained
+referencing==0.35.1
+
+# For conditions
+panzi-json-logic
+# For conditions - extracted number from a body of text
+price-parser
+
 # Scheduler - Windows seemed to miss a lot of default timezone info (even "UTC" !)
 tzdata
+
+#typing_extensions ==4.8.0
+
+pluggy ~= 1.5
+
+
+

test_processor_registration.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+
+from changedetectionio.processors import available_processors
+from changedetectionio.processors.processor_registry import get_processor_class, get_processor_form
+
+# Test processor registration
+print("=== Available Processors ===")
+processors = available_processors()
+for name, description in processors:
+    print(f"Processor: {name} - {description}")
+
+# Check if our WHOIS processor is registered
+whois_processor_name = "whois_processor"
+whois_found = any(name == whois_processor_name for name, _ in processors)
+
+if whois_found:
+    print(f"\nWHOIS Processor found! Getting processor class and form...")
+    
+    # Get the processor class
+    processor_class = get_processor_class(whois_processor_name)
+    print(f"Processor class: {processor_class}")
+    print(f"Processor class name: {processor_class.__name__ if processor_class else None}")
+    print(f"Processor class module: {processor_class.__module__ if processor_class else None}")
+    
+    # Get the processor form
+    processor_form = get_processor_form(whois_processor_name)
+    print(f"Processor form: {processor_form}")
+    
+    print("\nWHOIS Processor successfully registered")
+else:
+    print(f"\nWHOIS Processor not found in available processors")

test_processors.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+
+from changedetectionio.processors import available_processors
+from changedetectionio.processors import find_processors
+
+# Test traditional processor discovery
+print("=== Traditional Processor Discovery ===")
+traditional_processors = find_processors()
+for module, name in traditional_processors:
+    print(f"Found processor: {name} in {module.__name__}")
+
+# Test combined processor discovery (traditional + pluggy)
+print("\n=== Combined Processor Discovery ===")
+combined_processors = available_processors()
+for name, description in combined_processors:
+    print(f"Processor: {name} - {description}")

test_whois_extraction.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+import urllib.parse
+import re
+import sys
+
+def extract_domain_from_url(url):
+    """Extract domain from a URL"""
+    parsed_url = urllib.parse.urlparse(url)
+    domain = parsed_url.netloc
+    
+    # Remove www. prefix if present
+    domain = re.sub(r'^www\.', '', domain)
+    
+    return domain
+
+# Test domain extraction
+test_urls = [
+    "https://changedetection.io",
+    "http://www.example.com/page",
+    "https://subdomain.domain.co.uk/path?query=1",
+    "ftp://ftp.example.org",
+    "https://www.changedetection.io/page/subpage",
+]
+
+print("=== Domain Extraction Test ===")
+for url in test_urls:
+    domain = extract_domain_from_url(url)
+    print(f"URL: {url} -> Domain: {domain}")
+
+# Test WHOIS lookup for changedetection.io
+try:
+    import whois
+    
+    domain = extract_domain_from_url("https://changedetection.io")
+    print(f"\n=== WHOIS lookup for {domain} ===")
+    
+    whois_info = whois.whois(domain)
+    
+    # Print key information
+    print(f"Domain Name: {whois_info.get('domain_name', '')}")
+    print(f"Registrar: {whois_info.get('registrar', '')}")
+    print(f"Creation Date: {whois_info.get('creation_date', '')}")
+    print(f"Expiration Date: {whois_info.get('expiration_date', '')}")
+    
+    print("\nWHOIS lookup successful!")
+    
+except ImportError:
+    print("python-whois module not installed. Run: pip install python-whois")
+    sys.exit(1)
+except Exception as e:
+    print(f"Error performing WHOIS lookup: {str(e)}")
+    sys.exit(1)

test_whois_processor.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+from changedetectionio.processors import available_processors
+from changedetectionio.processors.processor_registry import get_processor_class
+import urllib.parse
+import sys
+
+# First, verify our processor is available
+print("=== Available Processors ===")
+processors = available_processors()
+for name, description in processors:
+    print(f"Processor: {name} - {description}")
+
+# Get the WHOIS processor class
+whois_processor_class = get_processor_class("whois_processor")
+if not whois_processor_class:
+    print("ERROR: WHOIS processor not found in available processors.")
+    sys.exit(1)
+
+print(f"\nFound WHOIS processor class: {whois_processor_class}")
+
+# Test the WHOIS processor directly
+try:
+    # Parse a domain from a URL
+    url = "https://changedetection.io"
+    parsed_url = urllib.parse.urlparse(url)
+    domain = parsed_url.netloc
+    
+    # Import whois and fetch information
+    import whois
+    whois_info = whois.whois(domain)
+    
+    print(f"\n=== WHOIS Information for {domain} ===")
+    
+    # Print the information
+    if hasattr(whois_info, 'text'):
+        print(whois_info.text)
+    else:
+        for key, value in whois_info.items():
+            if value:
+                print(f"{key}: {value}")
+                
+    print("\nSuccessfully retrieved WHOIS data!")
+    
+except Exception as e:
+    print(f"Error fetching WHOIS data: {str(e)}")
+    sys.exit(1)

test_whois_processor_full.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+
+from changedetectionio.processors import available_processors
+from changedetectionio.processors.processor_registry import get_processor_class
+import unittest
+import sys
+from unittest.mock import MagicMock, patch
+import urllib.parse
+
+# First, verify our processor is available
+print("=== Available Processors ===")
+processors = available_processors()
+for name, description in processors:
+    print(f"Processor: {name} - {description}")
+
+# Get the WHOIS processor class
+whois_processor_class = get_processor_class("whois_processor")
+if not whois_processor_class:
+    print("ERROR: WHOIS processor not found in available processors.")
+    sys.exit(1)
+
+print(f"\nFound WHOIS processor class: {whois_processor_class}")
+
+# Create a test for our WHOIS processor
+class TestWhoisProcessor(unittest.TestCase):
+    
+    # Use the real whois function - tests will actually make network requests
+    def test_whois_processor_real(self):
+        # Extract the domain from the URL
+        test_url = "https://changedetection.io"
+        parsed_url = urllib.parse.urlparse(test_url)
+        domain = parsed_url.netloc
+        
+        # Create a minimal mock datastore
+        mock_datastore = MagicMock()
+        mock_datastore.data = {
+            'watching': {'test-uuid': {'url': test_url}},
+            'settings': {
+                'application': {'empty_pages_are_a_change': False},
+                'requests': {'timeout': 30}
+            }
+        }
+        mock_datastore.get_all_base_headers.return_value = {}
+        mock_datastore.get_all_headers_in_textfile_for_watch.return_value = {}
+        mock_datastore.get_preferred_proxy_for_watch.return_value = None
+        mock_datastore.get_tag_overrides_for_watch.return_value = []
+        
+        # Create a minimal mock watch that mimics the real Watch class
+        class MockWatch:
+            def __init__(self, url):
+                self.link = url
+                self.is_pdf = False
+                self.has_browser_steps = False
+                self.is_source_type_url = False
+                self.history = {}
+                self.history_n = 0
+                self.last_viewed = 0
+                self.newest_history_key = 0
+                
+            def get(self, key, default=None):
+                if key == 'uuid':
+                    return 'test-uuid'
+                elif key == 'include_filters':
+                    return []
+                elif key == 'body':
+                    return None
+                elif key == 'method':
+                    return 'GET'
+                elif key == 'headers':
+                    return {}
+                elif key == 'browser_steps':
+                    return []
+                return default
+                
+            def __getitem__(self, key):
+                return self.get(key)
+                
+            def get_last_fetched_text_before_filters(self):
+                return ""
+            
+            def save_last_text_fetched_before_filters(self, content):
+                pass
+                
+            def has_special_diff_filter_options_set(self):
+                return False
+                
+            def lines_contain_something_unique_compared_to_history(self, lines, ignore_whitespace):
+                return True
+                
+        mock_watch = MockWatch(test_url)
+        
+        # Create a more complete mock fetcher
+        class MockFetcher:
+            def __init__(self):
+                self.content = ""
+                self.raw_content = b""
+                self.headers = {'Content-Type': 'text/plain'}
+                self.screenshot = None
+                self.xpath_data = None
+                self.instock_data = None
+                self.browser_steps = []
+            
+            def get_last_status_code(self):
+                return 200
+                
+            def get_all_headers(self):
+                return {'content-type': 'text/plain'}
+                
+            def quit(self):
+                pass
+                
+            def run(self, **kwargs):
+                pass
+                
+        # Create the processor and set the mock fetcher
+        processor = whois_processor_class(datastore=mock_datastore, watch_uuid='test-uuid')
+        processor.fetcher = MockFetcher()
+        
+        # Run the processor - this will make an actual WHOIS request
+        changed, update_obj, content = processor.run_changedetection(mock_watch)
+        
+        # Print the content for debugging
+        content_str = content.decode('utf-8')
+        print(f"\n=== WHOIS Content from processor (first 200 chars) ===")
+        print(content_str[:200] + "...")
+        
+        # Verify the content contains domain information
+        self.assertIn(domain, content_str)
+        self.assertIn("Domain Name", content_str)
+        self.assertIn("Creation Date", content_str)
+        
+        print("\nWHOIS processor test with real data PASSED!")
+
+# Run the test
+if __name__ == "__main__":
+    unittest.main(argv=['first-arg-is-ignored'], exit=False)

test_whois_simple.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+import urllib.parse
+import re
+import whois
+
+# Test with changedetection.io domain
+url = "https://changedetection.io"
+
+# Extract domain from URL
+parsed_url = urllib.parse.urlparse(url)
+domain = parsed_url.netloc
+
+# Remove www. prefix if present
+domain = re.sub(r'^www\.', '', domain)
+
+# Fetch WHOIS information
+print(f"Looking up WHOIS data for domain: {domain}")
+whois_info = whois.whois(domain)
+
+# Print key WHOIS data
+print("\nKey WHOIS information:")
+print(f"Domain Name: {whois_info.get('domain_name', 'Unknown')}")
+print(f"Registrar: {whois_info.get('registrar', 'Unknown')}")
+print(f"Creation Date: {whois_info.get('creation_date', 'Unknown')}")
+print(f"Expiration Date: {whois_info.get('expiration_date', 'Unknown')}")
+print(f"Updated Date: {whois_info.get('updated_date', 'Unknown')}")
+
+# Format as text
+whois_text = f"WHOIS Information for domain: {domain}\n\n"
+for key, value in whois_info.items():
+    if value:
+        whois_text += f"{key}: {value}\n"
+
+# Print the first 200 characters
+print("\nFormatted WHOIS data (first 200 chars):")
+print(whois_text[:200] + "...")
+
+print("\nWHOIS lookup successful!")