0.49.18

Co-authored-by: silversub <silversub@gmail.com>

.github/workflows/test-only.yml

@@ -8,13 +8,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Lint with flake8
+      - name: Lint with Ruff
         run: |
-          pip3 install flake8
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+          pip install ruff
+          # Check for syntax errors and undefined names
+          ruff check . --select E9,F63,F7,F82
+          # Complete check with errors treated as warnings
+          ruff check . --exit-zero
 
   test-application-3-10:
     needs: lint-code
@@ -41,5 +41,4 @@ jobs:
     uses: ./.github/workflows/test-stack-reusable-workflow.yml
     with:
       python-version: '3.13'
-      skip-pypuppeteer: true
-      
+      skip-pypuppeteer: true

.github/workflows/test-stack-reusable-workflow.yml

@@ -172,8 +172,8 @@ jobs:
           curl --retry-connrefused --retry 6  -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
 
           # Check whether TRACE log is enabled.
-          # Also, check whether TRACE is came from STDERR
-          docker logs test-changedetectionio 2>&1 1>/dev/null | grep 'TRACE log is enabled' || exit 1
+          # Also, check whether TRACE came from STDOUT
+          docker logs test-changedetectionio 2>/dev/null | grep 'TRACE log is enabled' || exit 1
           # Check whether DEBUG is came from STDOUT
           docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
 

.gitignore

@@ -16,6 +16,7 @@ dist/
 .env
 .venv/
 venv/
+.python-version
 
 # IDEs
 .idea

.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.11.2
+    hooks:
+      # Lint (and apply safe fixes)
+      - id: ruff
+        args: [--fix]
+      # Fomrat
+      - id: ruff-format

.ruff.toml

@@ -0,0 +1,48 @@
+# Minimum supported version
+target-version = "py310"
+
+# Formatting options
+line-length = 100
+indent-width = 4
+
+exclude = [
+    "__pycache__",
+    ".eggs",
+    ".git",
+    ".tox",
+    ".venv",
+    "*.egg-info",
+    "*.pyc",
+]
+
+[lint]
+# https://docs.astral.sh/ruff/rules/
+select = [
+    "B", # flake8-bugbear
+    "B9",
+    "C", 
+    "E", # pycodestyle
+    "F", # Pyflakes
+    "I", # isort
+    "N", # pep8-naming
+    "UP", # pyupgrade
+    "W", # pycodestyle
+]
+ignore = [
+    "B007", # unused-loop-control-variable
+    "B909", # loop-iterator-mutation
+    "E203", # whitespace-before-punctuation
+    "E266", # multiple-leading-hashes-for-block-comment
+    "E501", # redundant-backslash
+    "F403", # undefined-local-with-import-star
+    "N802", # invalid-function-name
+    "N806", # non-lowercase-variable-in-function
+    "N815", # mixed-case-variable-in-class-scope
+]
+
+[lint.mccabe]
+max-complexity = 12
+
+[format]
+indent-style = "space"
+quote-style = "preserve"

MANIFEST.in

@@ -5,6 +5,7 @@ recursive-include changedetectionio/conditions *
 recursive-include changedetectionio/model *
 recursive-include changedetectionio/notification *
 recursive-include changedetectionio/processors *
+recursive-include changedetectionio/realtime *
 recursive-include changedetectionio/static *
 recursive-include changedetectionio/templates *
 recursive-include changedetectionio/tests *

changedetectionio/PLUGIN_README.md

@@ -0,0 +1,98 @@
+# Creating Plugins for changedetection.io
+
+This document describes how to create plugins for changedetection.io. Plugins can be used to extend the functionality of the application in various ways.
+
+## Plugin Types
+
+### UI Stats Tab Plugins
+
+These plugins can add content to the Stats tab in the Edit page. This is useful for adding custom statistics or visualizations about a watch.
+
+#### Creating a UI Stats Tab Plugin
+
+1. Create a Python file in a directory that will be loaded by the plugin system.
+
+2. Use the `global_hookimpl` decorator to implement the `ui_edit_stats_extras` hook:
+
+```python
+import pluggy
+from loguru import logger
+
+global_hookimpl = pluggy.HookimplMarker("changedetectionio")
+
+@global_hookimpl
+def ui_edit_stats_extras(watch):
+    """Add custom content to the stats tab"""
+    # Calculate or retrieve your stats
+    my_stat = calculate_something(watch)
+    
+    # Return HTML content as a string
+    html = f"""
+    <div class="my-plugin-stats">
+        <h4>My Plugin Statistics</h4>
+        <p>My statistic: {my_stat}</p>
+    </div>
+    """
+    return html
+```
+
+3. The HTML you return will be included in the Stats tab.
+
+## Plugin Loading
+
+Plugins can be loaded from:
+
+1. Built-in plugin directories in the codebase
+2. External packages using setuptools entry points
+
+To add a new plugin directory, modify the `plugin_dirs` dictionary in `pluggy_interface.py`.
+
+## Example Plugin
+
+Here's a simple example of a plugin that adds a word count statistic to the Stats tab:
+
+```python
+import pluggy
+from loguru import logger
+
+global_hookimpl = pluggy.HookimplMarker("changedetectionio")
+
+def count_words_in_history(watch):
+    """Count words in the latest snapshot"""
+    try:
+        if not watch.history.keys():
+            return 0
+            
+        latest_key = list(watch.history.keys())[-1]
+        latest_content = watch.get_history_snapshot(latest_key)
+        return len(latest_content.split())
+    except Exception as e:
+        logger.error(f"Error counting words: {str(e)}")
+        return 0
+
+@global_hookimpl
+def ui_edit_stats_extras(watch):
+    """Add word count to the Stats tab"""
+    word_count = count_words_in_history(watch)
+    
+    html = f"""
+    <div class="word-count-stats">
+        <h4>Content Analysis</h4>
+        <table class="pure-table">
+            <tbody>
+                <tr>
+                    <td>Word count (latest snapshot)</td>
+                    <td>{word_count}</td>
+                </tr>
+            </tbody>
+        </table>
+    </div>
+    """
+    return html
+```
+
+## Testing Your Plugin
+
+1. Place your plugin in one of the directories scanned by the plugin system
+2. Restart changedetection.io
+3. Go to the Edit page of a watch and check the Stats tab to see your content

changedetectionio/__init__.py

@@ -2,19 +2,17 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.49.13'
+__version__ = '0.49.18'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
 import os
-os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
-import eventlet
-import eventlet.wsgi
 import getopt
 import platform
 import signal
 import socket
 import sys
+from werkzeug.serving import run_simple
 
 from changedetectionio import store
 from changedetectionio.flask_app import changedetection_app
@@ -33,8 +31,17 @@ def sigshutdown_handler(_signo, _stack_frame):
     logger.critical(f'Shutdown: Got Signal - {name} ({_signo}), Saving DB to disk and calling shutdown')
     datastore.sync_to_json()
     logger.success('Sync JSON to disk complete.')
-    # This will throw a SystemExit exception, because eventlet.wsgi.server doesn't know how to deal with it.
-    # Solution: move to gevent or other server in the future (#2014)
+    
+    # Shutdown socketio server if available
+    from changedetectionio.flask_app import socketio_server
+    if socketio_server and hasattr(socketio_server, 'shutdown'):
+        try:
+            logger.info("Shutting down Socket.IO server...")
+            socketio_server.shutdown()
+        except Exception as e:
+            logger.error(f"Error shutting down Socket.IO server: {str(e)}")
+    
+    # Set flags for clean shutdown
     datastore.stop_thread = True
     app.config.exit.set()
     sys.exit()
@@ -106,7 +113,7 @@ def main():
     # Without this, a logger will be duplicated
     logger.remove()
     try:
-        log_level_for_stdout = { 'DEBUG', 'SUCCESS' }
+        log_level_for_stdout = { 'TRACE', 'DEBUG', 'INFO', 'SUCCESS' }
         logger.configure(handlers=[
             {"sink": sys.stdout, "level": logger_level,
              "filter" : lambda record: record['level'].name in log_level_for_stdout},
@@ -196,13 +203,85 @@ def main():
 
     s_type = socket.AF_INET6 if ipv6_enabled else socket.AF_INET
 
-    if ssl_mode:
-        # @todo finalise SSL config, but this should get you in the right direction if you need it.
-        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port), s_type),
-                                               certfile='cert.pem',
-                                               keyfile='privkey.pem',
-                                               server_side=True), app)
+    # Get socketio_server from flask_app
+    from changedetectionio.flask_app import socketio_server
 
+    if socketio_server and datastore.data['settings']['application']['ui'].get('open_diff_in_new_tab'):
+        logger.info("Starting server with Socket.IO support (using threading)...")
+
+        # Use Flask-SocketIO's run method with error handling for Werkzeug warning
+        # This is the cleanest approach that works with all Flask-SocketIO versions
+        # Use '0.0.0.0' as the default host if none is specified
+        # This will listen on all available interfaces
+        listen_host = '0.0.0.0' if host == '' else host
+        logger.info(f"Using host: {listen_host} and port: {port}")
+
+        try:
+            # First try with the allow_unsafe_werkzeug parameter (newer versions)
+            if ssl_mode:
+                socketio_server.run(
+                    app,
+                    host=listen_host,
+                    port=int(port),
+                    certfile='cert.pem',
+                    keyfile='privkey.pem',
+                    debug=False,
+                    use_reloader=False,
+                    allow_unsafe_werkzeug=True  # Only in newer versions
+                )
+            else:
+                socketio_server.run(
+                    app,
+                    host=listen_host,
+                    port=int(port),
+                    debug=False,
+                    use_reloader=False,
+                    allow_unsafe_werkzeug=True  # Only in newer versions
+                )
+        except TypeError:
+            # If allow_unsafe_werkzeug is not a valid parameter, try without it
+            logger.info("Falling back to basic run method without allow_unsafe_werkzeug")
+            # Override the werkzeug safety check by setting an environment variable
+            os.environ['WERKZEUG_RUN_MAIN'] = 'true'
+            if ssl_mode:
+                socketio_server.run(
+                    app,
+                    host=listen_host,
+                    port=int(port),
+                    certfile='cert.pem',
+                    keyfile='privkey.pem',
+                    debug=False,
+                    use_reloader=False
+                )
+            else:
+                socketio_server.run(
+                    app,
+                    host=listen_host,
+                    port=int(port),
+                    debug=False,
+                    use_reloader=False
+                )
     else:
-        eventlet.wsgi.server(eventlet.listen((host, int(port)), s_type), app)
+        logger.warning("Socket.IO server not initialized, falling back to standard WSGI server")
+        # Fallback to standard WSGI server if socketio_server is not available
+        listen_host = '0.0.0.0' if host == '' else host
+        if ssl_mode:
+            # Use Werkzeug's run_simple with SSL support
+            run_simple(
+                hostname=listen_host,
+                port=int(port),
+                application=app,
+                use_reloader=False,
+                use_debugger=False,
+                ssl_context=('cert.pem', 'privkey.pem')
+            )
+        else:
+            # Use Werkzeug's run_simple for standard HTTP
+            run_simple(
+                hostname=listen_host,
+                port=int(port),
+                application=app,
+                use_reloader=False,
+                use_debugger=False
+            )
 

changedetectionio/blueprint/browser_steps/__init__.py

@@ -53,14 +53,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         a = "?" if not '?' in base_url else '&'
         base_url += a + f"timeout={keepalive_ms}"
 
-        try:
-            browsersteps_start_session['browser'] = io_interface_context.chromium.connect_over_cdp(base_url)
-        except Exception as e:
-            if 'ECONNREFUSED' in str(e):
-                return make_response('Unable to start the Playwright Browser session, is it running?', 401)
-            else:
-                # Other errors, bad URL syntax, bad reply etc
-                return make_response(str(e), 401)
+        browsersteps_start_session['browser'] = io_interface_context.chromium.connect_over_cdp(base_url)
 
         proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
         proxy = None
@@ -109,7 +102,16 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         logger.debug("Starting connection with playwright")
         logger.debug("browser_steps.py connecting")
-        browsersteps_sessions[browsersteps_session_id] = start_browsersteps_session(watch_uuid)
+
+        try:
+            browsersteps_sessions[browsersteps_session_id] = start_browsersteps_session(watch_uuid)
+        except Exception as e:
+            if 'ECONNREFUSED' in str(e):
+                return make_response('Unable to start the Playwright Browser session, is sockpuppetbrowser running? Network configuration is OK?', 401)
+            else:
+                # Other errors, bad URL syntax, bad reply etc
+                return make_response(str(e), 401)
+
         logger.debug("Starting connection with playwright - done")
         return {'browsersteps_session_id': browsersteps_session_id}
 
@@ -166,9 +168,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             step_optional_value = request.form.get('optional_value')
             is_last_step = strtobool(request.form.get('is_last_step'))
 
-            # @todo try.. accept.. nice errors not popups..
             try:
-
                 browsersteps_sessions[browsersteps_session_id]['browserstepper'].call_action(action_name=step_operation,
                                          selector=step_selector,
                                          optional_value=step_optional_value)

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -1,6 +1,8 @@
 import os
 import time
 import re
+import sys
+import traceback
 from random import randint
 from loguru import logger
 
@@ -35,6 +37,7 @@ browser_step_ui_config = {'Choose one': '0 0',
                           'Make all child elements visible': '1 0',
                           'Press Enter': '0 0',
                           'Select by label': '1 1',
+                          '<select> by option text': '1 1',
                           'Scroll down': '0 0',
                           'Uncheck checkbox': '1 0',
                           'Wait for seconds': '0 1',
@@ -54,7 +57,6 @@ browser_step_ui_config = {'Choose one': '0 0',
 class steppable_browser_interface():
     page = None
     start_url = None
-
     action_timeout = 10 * 1000
 
     def __init__(self, start_url):
@@ -62,6 +64,10 @@ class steppable_browser_interface():
 
     # Convert and perform "Click Button" for example
     def call_action(self, action_name, selector=None, optional_value=None):
+        if self.page is None:
+            logger.warning("Cannot call action on None page object")
+            return
+            
         now = time.time()
         call_action_name = re.sub('[^0-9a-zA-Z]+', '_', action_name.lower())
         if call_action_name == 'choose_one':
@@ -72,28 +78,33 @@ class steppable_browser_interface():
         if selector and selector.startswith('/') and not selector.startswith('//'):
             selector = "xpath=" + selector
 
+        # Check if action handler exists
+        if not hasattr(self, "action_" + call_action_name):
+            logger.warning(f"Action handler for '{call_action_name}' not found")
+            return
+            
         action_handler = getattr(self, "action_" + call_action_name)
 
         # Support for Jinja2 variables in the value and selector
-
         if selector and ('{%' in selector or '{{' in selector):
             selector = jinja_render(template_str=selector)
 
         if optional_value and ('{%' in optional_value or '{{' in optional_value):
             optional_value = jinja_render(template_str=optional_value)
 
+
         action_handler(selector, optional_value)
+        # Safely wait for timeout
         self.page.wait_for_timeout(1.5 * 1000)
         logger.debug(f"Call action done in {time.time()-now:.2f}s")
 
     def action_goto_url(self, selector=None, value=None):
-        # self.page.set_viewport_size({"width": 1280, "height": 5000})
+        if not value:
+            logger.warning("No URL provided for goto_url action")
+            return None
+            
         now = time.time()
         response = self.page.goto(value, timeout=0, wait_until='load')
-        # Should be the same as the puppeteer_fetch.js methods, means, load with no timeout set (skip timeout)
-        #and also wait for seconds ?
-        #await page.waitForTimeout(1000);
-        #await page.waitForTimeout(extra_wait_ms);
         logger.debug(f"Time to goto URL {time.time()-now:.2f}s")
         return response
 
@@ -103,36 +114,40 @@ class steppable_browser_interface():
 
     def action_click_element_containing_text(self, selector=None, value=''):
         logger.debug("Clicking element containing text")
-        if not len(value.strip()):
+        if not value or not len(value.strip()):
             return
+            
         elem = self.page.get_by_text(value)
         if elem.count():
             elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
 
+
     def action_click_element_containing_text_if_exists(self, selector=None, value=''):
         logger.debug("Clicking element containing text if exists")
-        if not len(value.strip()):
+        if not value or not len(value.strip()):
             return
+            
         elem = self.page.get_by_text(value)
         logger.debug(f"Clicking element containing text - {elem.count()} elements found")
         if elem.count():
             elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
-        else:
-            return
+                
 
     def action_enter_text_in_field(self, selector, value):
-        if not len(selector.strip()):
+        if not selector or not len(selector.strip()):
             return
 
         self.page.fill(selector, value, timeout=self.action_timeout)
 
     def action_execute_js(self, selector, value):
-        response = self.page.evaluate(value)
-        return response
+        if not value:
+            return None
+            
+        return self.page.evaluate(value)
 
     def action_click_element(self, selector, value):
         logger.debug("Clicking element")
-        if not len(selector.strip()):
+        if not selector or not len(selector.strip()):
             return
 
         self.page.click(selector=selector, timeout=self.action_timeout + 20 * 1000, delay=randint(200, 500))
@@ -140,24 +155,38 @@ class steppable_browser_interface():
     def action_click_element_if_exists(self, selector, value):
         import playwright._impl._errors as _api_types
         logger.debug("Clicking element if exists")
-        if not len(selector.strip()):
+        if not selector or not len(selector.strip()):
             return
+            
         try:
             self.page.click(selector, timeout=self.action_timeout, delay=randint(200, 500))
-        except _api_types.TimeoutError as e:
+        except _api_types.TimeoutError:
             return
-        except _api_types.Error as e:
+        except _api_types.Error:
             # Element was there, but page redrew and now its long long gone
             return
+                
 
     def action_click_x_y(self, selector, value):
-        if not re.match(r'^\s?\d+\s?,\s?\d+\s?$', value):
-            raise Exception("'Click X,Y' step should be in the format of '100 , 90'")
+        if not value or not re.match(r'^\s?\d+\s?,\s?\d+\s?$', value):
+            logger.warning("'Click X,Y' step should be in the format of '100 , 90'")
+            return
 
-        x, y = value.strip().split(',')
-        x = int(float(x.strip()))
-        y = int(float(y.strip()))
-        self.page.mouse.click(x=x, y=y, delay=randint(200, 500))
+        try:
+            x, y = value.strip().split(',')
+            x = int(float(x.strip()))
+            y = int(float(y.strip()))
+            
+            self.page.mouse.click(x=x, y=y, delay=randint(200, 500))
+                
+        except Exception as e:
+            logger.error(f"Error parsing x,y coordinates: {str(e)}")
+
+    def action__select_by_option_text(self, selector, value):
+        if not selector or not len(selector.strip()):
+            return
+
+        self.page.select_option(selector, label=value, timeout=self.action_timeout)
 
     def action_scroll_down(self, selector, value):
         # Some sites this doesnt work on for some reason
@@ -165,23 +194,42 @@ class steppable_browser_interface():
         self.page.wait_for_timeout(1000)
 
     def action_wait_for_seconds(self, selector, value):
-        self.page.wait_for_timeout(float(value.strip()) * 1000)
+        try:
+            seconds = float(value.strip()) if value else 1.0
+            self.page.wait_for_timeout(seconds * 1000)
+        except (ValueError, TypeError) as e:
+            logger.error(f"Invalid value for wait_for_seconds: {str(e)}")
 
     def action_wait_for_text(self, selector, value):
+        if not value:
+            return
+            
         import json
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=30000)
+        self.page.wait_for_function(
+            f'document.querySelector("body").innerText.includes({v});',
+            timeout=30000
+        )
+            
 
     def action_wait_for_text_in_element(self, selector, value):
+        if not selector or not value:
+            return
+            
         import json
         s = json.dumps(selector)
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=30000)
+        
+        self.page.wait_for_function(
+            f'document.querySelector({s}).innerText.includes({v});',
+            timeout=30000
+        )
 
     # @todo - in the future make some popout interface to capture what needs to be set
     # https://playwright.dev/python/docs/api/class-keyboard
     def action_press_enter(self, selector, value):
         self.page.keyboard.press("Enter", delay=randint(200, 500))
+            
 
     def action_press_page_up(self, selector, value):
         self.page.keyboard.press("PageUp", delay=randint(200, 500))
@@ -190,17 +238,30 @@ class steppable_browser_interface():
         self.page.keyboard.press("PageDown", delay=randint(200, 500))
 
     def action_check_checkbox(self, selector, value):
+        if not selector:
+            return
+
         self.page.locator(selector).check(timeout=self.action_timeout)
 
     def action_uncheck_checkbox(self, selector, value):
+        if not selector:
+            return
+            
         self.page.locator(selector).uncheck(timeout=self.action_timeout)
+            
 
     def action_remove_elements(self, selector, value):
         """Removes all elements matching the given selector from the DOM."""
+        if not selector:
+            return
+            
         self.page.locator(selector).evaluate_all("els => els.forEach(el => el.remove())")
 
     def action_make_all_child_elements_visible(self, selector, value):
         """Recursively makes all child elements inside the given selector fully visible."""
+        if not selector:
+            return
+            
         self.page.locator(selector).locator("*").evaluate_all("""
             els => els.forEach(el => {
                 el.style.display = 'block';   // Forces it to be displayed
@@ -224,7 +285,9 @@ class browsersteps_live_ui(steppable_browser_interface):
     # bump and kill this if idle after X sec
     age_start = 0
     headers = {}
-
+    # Track if resources are properly cleaned up
+    _is_cleaned_up = False
+    
     # use a special driver, maybe locally etc
     command_executor = os.getenv(
         "PLAYWRIGHT_BROWSERSTEPS_DRIVER_URL"
@@ -243,9 +306,14 @@ class browsersteps_live_ui(steppable_browser_interface):
         self.age_start = time.time()
         self.playwright_browser = playwright_browser
         self.start_url = start_url
+        self._is_cleaned_up = False
         if self.context is None:
             self.connect(proxy=proxy)
 
+    def __del__(self):
+        # Ensure cleanup happens if object is garbage collected
+        self.cleanup()
+
     # Connect and setup a new context
     def connect(self, proxy=None):
         # Should only get called once - test that
@@ -264,31 +332,74 @@ class browsersteps_live_ui(steppable_browser_interface):
             user_agent=manage_user_agent(headers=self.headers),
         )
 
-
         self.page = self.context.new_page()
 
         # self.page.set_default_navigation_timeout(keep_open)
         self.page.set_default_timeout(keep_open)
-        # @todo probably this doesnt work
-        self.page.on(
-            "close",
-            self.mark_as_closed,
-        )
+        # Set event handlers
+        self.page.on("close", self.mark_as_closed)
         # Listen for all console events and handle errors
         self.page.on("console", lambda msg: print(f"Browser steps console - {msg.type}: {msg.text} {msg.args}"))
 
         logger.debug(f"Time to browser setup {time.time()-now:.2f}s")
         self.page.wait_for_timeout(1 * 1000)
 
-
     def mark_as_closed(self):
         logger.debug("Page closed, cleaning up..")
+        self.cleanup()
+
+    def cleanup(self):
+        """Properly clean up all resources to prevent memory leaks"""
+        if self._is_cleaned_up:
+            return
+            
+        logger.debug("Cleaning up browser steps resources")
+        
+        # Clean up page
+        if hasattr(self, 'page') and self.page is not None:
+            try:
+                # Force garbage collection before closing
+                self.page.request_gc()
+            except Exception as e:
+                logger.debug(f"Error during page garbage collection: {str(e)}")
+                
+            try:
+                # Remove event listeners before closing
+                self.page.remove_listener("close", self.mark_as_closed)
+            except Exception as e:
+                logger.debug(f"Error removing event listeners: {str(e)}")
+                
+            try:
+                self.page.close()
+            except Exception as e:
+                logger.debug(f"Error closing page: {str(e)}")
+            
+            self.page = None
+
+        # Clean up context
+        if hasattr(self, 'context') and self.context is not None:
+            try:
+                self.context.close()
+            except Exception as e:
+                logger.debug(f"Error closing context: {str(e)}")
+            
+            self.context = None
+            
+        self._is_cleaned_up = True
+        logger.debug("Browser steps resources cleanup complete")
 
     @property
     def has_expired(self):
-        if not self.page:
+        if not self.page or self._is_cleaned_up:
             return True
-
+        
+        # Check if session has expired based on age
+        max_age_seconds = int(os.getenv("BROWSER_STEPS_MAX_AGE_SECONDS", 60 * 10))  # Default 10 minutes
+        if (time.time() - self.age_start) > max_age_seconds:
+            logger.debug(f"Browser steps session expired after {max_age_seconds} seconds")
+            return True
+            
+        return False
 
     def get_current_state(self):
         """Return the screenshot and interactive elements mapping, generally always called after action_()"""
@@ -297,36 +408,55 @@ class browsersteps_live_ui(steppable_browser_interface):
         # because we for now only run browser steps in playwright mode (not puppeteer mode)
         from changedetectionio.content_fetchers.playwright import capture_full_page
 
+        # Safety check - don't proceed if resources are cleaned up
+        if self._is_cleaned_up or self.page is None:
+            logger.warning("Attempted to get current state after cleanup")
+            return (None, None)
+
         xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
 
         now = time.time()
         self.page.wait_for_timeout(1 * 1000)
 
-        screenshot = capture_full_page(page=self.page)
+        screenshot = None
+        xpath_data = None
+        
+        try:
+            # Get screenshot first
+            screenshot = capture_full_page(page=self.page)
+            logger.debug(f"Time to get screenshot from browser {time.time() - now:.2f}s")
 
-        logger.debug(f"Time to get screenshot from browser {time.time() - now:.2f}s")
+            # Then get interactive elements
+            now = time.time()
+            self.page.evaluate("var include_filters=''")
+            self.page.request_gc()
 
-        now = time.time()
-        self.page.evaluate("var include_filters=''")
-        # Go find the interactive elements
-        # @todo in the future, something smarter that can scan for elements with .click/focus etc event handlers?
+            scan_elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
 
-        self.page.request_gc()
+            MAX_TOTAL_HEIGHT = int(os.getenv("SCREENSHOT_MAX_HEIGHT", SCREENSHOT_MAX_HEIGHT_DEFAULT))
+            xpath_data = json.loads(self.page.evaluate(xpath_element_js, {
+                "visualselector_xpath_selectors": scan_elements,
+                "max_height": MAX_TOTAL_HEIGHT
+            }))
+            self.page.request_gc()
 
-        scan_elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
-
-        MAX_TOTAL_HEIGHT = int(os.getenv("SCREENSHOT_MAX_HEIGHT", SCREENSHOT_MAX_HEIGHT_DEFAULT))
-        xpath_data = json.loads(self.page.evaluate(xpath_element_js, {
-            "visualselector_xpath_selectors": scan_elements,
-            "max_height": MAX_TOTAL_HEIGHT
-        }))
-        self.page.request_gc()
-
-        # So the JS will find the smallest one first
-        xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
-        logger.debug(f"Time to scrape xPath element data in browser {time.time()-now:.2f}s")
-
-        # playwright._impl._api_types.Error: Browser closed.
-        # @todo show some countdown timer?
+            # Sort elements by size
+            xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
+            logger.debug(f"Time to scrape xPath element data in browser {time.time()-now:.2f}s")
+            
+        except Exception as e:
+            logger.error(f"Error getting current state: {str(e)}")
+            # Attempt recovery - force garbage collection
+            try:
+                self.page.request_gc()
+            except:
+                pass
+        
+        # Request garbage collection one final time
+        try:
+            self.page.request_gc()
+        except:
+            pass
+            
         return (screenshot, xpath_data)
 

changedetectionio/blueprint/settings/templates/settings.html

@@ -246,6 +246,10 @@ nav
                     {{ render_checkbox_field(form.application.form.ui.form.open_diff_in_new_tab, class="open_diff_in_new_tab") }}
                     <span class="pure-form-message-inline">Enable this setting to open the diff page in a new tab. If disabled, the diff page will open in the current tab.</span>
                 </div>
+                <div class="pure-control-group">
+                    <span class="pure-form-message-inline">Enable realtime updates in the UI</span>
+                </div>
+
             </div>
             <div class="tab-pane-inner" id="proxies">
                 <div id="recommended-proxy">

changedetectionio/blueprint/ui/__init__.py

@@ -3,12 +3,13 @@ from flask import Blueprint, request, redirect, url_for, flash, render_template,
 from loguru import logger
 from functools import wraps
 
+from changedetectionio.blueprint.ui.ajax import constuct_ui_ajax_blueprint
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.blueprint.ui.edit import construct_blueprint as construct_edit_blueprint
 from changedetectionio.blueprint.ui.notification import construct_blueprint as construct_notification_blueprint
 from changedetectionio.blueprint.ui.views import construct_blueprint as construct_views_blueprint
 
-def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_update_threads, queuedWatchMetaData):
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_update_threads, queuedWatchMetaData, watch_check_update):
     ui_blueprint = Blueprint('ui', __name__, template_folder="templates")
     
     # Register the edit blueprint
@@ -20,9 +21,12 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
     ui_blueprint.register_blueprint(notification_blueprint)
     
     # Register the views blueprint
-    views_blueprint = construct_views_blueprint(datastore, update_q, queuedWatchMetaData)
+    views_blueprint = construct_views_blueprint(datastore, update_q, queuedWatchMetaData, watch_check_update)
     ui_blueprint.register_blueprint(views_blueprint)
-    
+
+    ui_ajax_blueprint = constuct_ui_ajax_blueprint(datastore, update_q, running_update_threads, queuedWatchMetaData, watch_check_update)
+    ui_blueprint.register_blueprint(ui_ajax_blueprint)
+
     # Import the login decorator
     from changedetectionio.auth_decorator import login_optionally_required
 
@@ -35,7 +39,6 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
             flash('Watch not found', 'error')
         else:
             flash("Cleared snapshot history for watch {}".format(uuid))
-
         return redirect(url_for('watchlist.index'))
 
     @ui_blueprint.route("/clear_history", methods=['GET', 'POST'])
@@ -47,7 +50,6 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
             if confirmtext == 'clear':
                 for uuid in datastore.data['watching'].keys():
                     datastore.clear_watch_history(uuid)
-
                 flash("Cleared snapshot history for all watches")
             else:
                 flash('Incorrect confirmation text.', 'error')
@@ -153,53 +155,46 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
     @login_optionally_required
     def form_watch_list_checkbox_operations():
         op = request.form['op']
-        uuids = request.form.getlist('uuids')
+        uuids = [u.strip() for u in request.form.getlist('uuids') if u]
 
         if (op == 'delete'):
             for uuid in uuids:
-                uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
-                    datastore.delete(uuid.strip())
+                    datastore.delete(uuid)
             flash("{} watches deleted".format(len(uuids)))
 
         elif (op == 'pause'):
             for uuid in uuids:
-                uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
-                    datastore.data['watching'][uuid.strip()]['paused'] = True
+                    datastore.data['watching'][uuid]['paused'] = True
             flash("{} watches paused".format(len(uuids)))
 
         elif (op == 'unpause'):
             for uuid in uuids:
-                uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
                     datastore.data['watching'][uuid.strip()]['paused'] = False
             flash("{} watches unpaused".format(len(uuids)))
 
         elif (op == 'mark-viewed'):
             for uuid in uuids:
-                uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
                     datastore.set_last_viewed(uuid, int(time.time()))
             flash("{} watches updated".format(len(uuids)))
 
         elif (op == 'mute'):
             for uuid in uuids:
-                uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
-                    datastore.data['watching'][uuid.strip()]['notification_muted'] = True
+                    datastore.data['watching'][uuid]['notification_muted'] = True
             flash("{} watches muted".format(len(uuids)))
 
         elif (op == 'unmute'):
             for uuid in uuids:
-                uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
-                    datastore.data['watching'][uuid.strip()]['notification_muted'] = False
+                    datastore.data['watching'][uuid]['notification_muted'] = False
             flash("{} watches un-muted".format(len(uuids)))
 
         elif (op == 'recheck'):
             for uuid in uuids:
-                uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
                     # Recheck and require a full reprocessing
                     update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
@@ -207,14 +202,12 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
 
         elif (op == 'clear-errors'):
             for uuid in uuids:
-                uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
                     datastore.data['watching'][uuid]["last_error"] = False
             flash(f"{len(uuids)} watches errors cleared")
 
         elif (op == 'clear-history'):
             for uuid in uuids:
-                uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
                     datastore.clear_watch_history(uuid)
             flash("{} watches cleared/reset.".format(len(uuids)))
@@ -224,12 +217,11 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
                 default_notification_format_for_watch
             )
             for uuid in uuids:
-                uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
-                    datastore.data['watching'][uuid.strip()]['notification_title'] = None
-                    datastore.data['watching'][uuid.strip()]['notification_body'] = None
-                    datastore.data['watching'][uuid.strip()]['notification_urls'] = []
-                    datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch
+                    datastore.data['watching'][uuid]['notification_title'] = None
+                    datastore.data['watching'][uuid]['notification_body'] = None
+                    datastore.data['watching'][uuid]['notification_urls'] = []
+                    datastore.data['watching'][uuid]['notification_format'] = default_notification_format_for_watch
             flash("{} watches set to use default notification settings".format(len(uuids)))
 
         elif (op == 'assign-tag'):
@@ -238,7 +230,6 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
                 tag_uuid = datastore.add_tag(title=op_extradata)
                 if op_extradata and tag_uuid:
                     for uuid in uuids:
-                        uuid = uuid.strip()
                         if datastore.data['watching'].get(uuid):
                             # Bug in old versions caused by bad edit page/tag handler
                             if isinstance(datastore.data['watching'][uuid]['tags'], str):
@@ -248,6 +239,11 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
 
             flash(f"{len(uuids)} watches were tagged")
 
+        if uuids:
+            for uuid in uuids:
+#                with app.app_context():
+                watch_check_update.send(watch_uuid=uuid)
+
         return redirect(url_for('watchlist.index'))
 
 

changedetectionio/blueprint/ui/ajax.py

@@ -0,0 +1,35 @@
+import time
+
+from blinker import signal
+from flask import Blueprint, request, redirect, url_for, flash, render_template, session
+
+
+from changedetectionio.store import ChangeDetectionStore
+
+def constuct_ui_ajax_blueprint(datastore: ChangeDetectionStore, update_q, running_update_threads, queuedWatchMetaData, watch_check_update):
+    ui_ajax_blueprint = Blueprint('ajax', __name__, template_folder="templates", url_prefix='/ajax')
+
+    # Import the login decorator
+    from changedetectionio.auth_decorator import login_optionally_required
+
+    @ui_ajax_blueprint.route("/toggle", methods=['POST'])
+    @login_optionally_required
+    def ajax_toggler():
+        op = request.values.get('op')
+        uuid = request.values.get('uuid')
+        if op and datastore.data['watching'].get(uuid):
+            if op == 'pause':
+                datastore.data['watching'][uuid].toggle_pause()
+            elif op == 'mute':
+                datastore.data['watching'][uuid].toggle_mute()
+            elif op == 'recheck':
+                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+
+            watch_check_update = signal('watch_check_update')
+            if watch_check_update:
+                watch_check_update.send(watch_uuid=uuid)
+
+        return 'OK'
+
+
+    return ui_ajax_blueprint

changedetectionio/blueprint/ui/edit.py

@@ -233,6 +233,9 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
             # Only works reliably with Playwright
 
+            # Import the global plugin system
+            from changedetectionio.pluggy_interface import collect_ui_edit_stats_extras
+            
             template_args = {
                 'available_processors': processors.available_processors(),
                 'available_timezones': sorted(available_timezones()),
@@ -250,6 +253,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                 'settings_application': datastore.data['settings']['application'],
                 'system_has_playwright_configured': os.getenv('PLAYWRIGHT_DRIVER_URL'),
                 'system_has_webdriver_configured': os.getenv('WEBDRIVER_URL'),
+                'ui_edit_stats_extras': collect_ui_edit_stats_extras(watch),
                 'visual_selector_data_ready': datastore.visualselector_data_is_ready(watch_uuid=uuid),
                 'timezone_default_config': datastore.data['settings']['application'].get('timezone'),
                 'using_global_webdriver_wait': not default['webdriver_delay'],

changedetectionio/blueprint/ui/views.py

@@ -8,7 +8,7 @@ from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.auth_decorator import login_optionally_required
 from changedetectionio import html_tools
 
-def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData, watch_check_update):
     views_blueprint = Blueprint('ui_views', __name__, template_folder="../ui/templates")
     
     @views_blueprint.route("/preview/<string:uuid>", methods=['GET'])

changedetectionio/blueprint/watchlist/__init__.py

@@ -72,31 +72,33 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                                 per_page=datastore.data['settings']['application'].get('pager_size', 50), css_framework="semantic")
 
         sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
+
         output = render_template(
             "watch-overview.html",
-                                 active_tag=active_tag,
-                                 active_tag_uuid=active_tag_uuid,
-                                 app_rss_token=datastore.data['settings']['application'].get('rss_access_token'),
-                                 datastore=datastore,
-                                 errored_count=errored_count,
-                                 form=form,
-                                 guid=datastore.data['app_guid'],
-                                 has_proxies=datastore.proxy_list,
-                                 has_unviewed=datastore.has_unviewed,
-                                 hosted_sticky=os.getenv("SALTED_PASS", False) == False,
-                                 now_time_server=time.time(),
-                                 pagination=pagination,
-                                 queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
-                                 search_q=request.args.get('q', '').strip(),
-                                 sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
-                                 sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
-                                 system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
-                                 tags=sorted_tags,
-                                 watches=sorted_watches
-                                 )
+            active_tag=active_tag,
+            active_tag_uuid=active_tag_uuid,
+            app_rss_token=datastore.data['settings']['application'].get('rss_access_token'),
+            ajax_toggle_url=url_for('ui.ajax.ajax_toggler'),
+            datastore=datastore,
+            errored_count=errored_count,
+            form=form,
+            guid=datastore.data['app_guid'],
+            has_proxies=datastore.proxy_list,
+            has_unviewed=datastore.has_unviewed,
+            hosted_sticky=os.getenv("SALTED_PASS", False) == False,
+            now_time_server=round(time.time()),
+            pagination=pagination,
+            queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
+            search_q=request.args.get('q', '').strip(),
+            sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
+            sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
+            system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
+            tags=sorted_tags,
+            watches=sorted_watches
+        )
 
         if session.get('share-link'):
-            del(session['share-link'])
+            del (session['share-link'])
 
         resp = make_response(output)
 

changedetectionio/blueprint/watchlist/templates/watch-overview.html

@@ -4,6 +4,7 @@
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 <script>let nowtimeserver={{ now_time_server }};</script>
+<script>let ajax_toggle_url="{{ ajax_toggle_url }}";</script>
 
 <style>
 .checking-now .last-checked {
@@ -100,59 +101,41 @@
             {% endif %}
             {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) %}
 
-                {% set is_unviewed = watch.newest_history_key| int > watch.last_viewed and watch.history_n>=2 %}
                 {% set checking_now = is_checking_now(watch) %}
-            <tr id="{{ watch.uuid }}"
+            <tr id="{{ watch.uuid }}" data-watch-uuid="{{ watch.uuid }}"
                 class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }} processor-{{ watch['processor'] }}
-                {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
-                {% if watch.last_notification_error is defined and watch.last_notification_error != False %}error{% endif %}
+                {# realtime.js also sets these vars on the row for update #}
+                {% if watch.compile_error_texts()|length >2 %}has-error{% endif %}
                 {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
-                {% if is_unviewed %}unviewed{% endif %}
+                {% if watch.has_unviewed %}unviewed{% endif %}
                 {% if watch.has_restock_info %} has-restock-info {% if watch['restock']['in_stock'] %}in-stock{% else %}not-in-stock{% endif %} {% else %}no-restock-info{% endif %}
                 {% if watch.uuid in queued_uuids %}queued{% endif %}
                 {% if checking_now %}checking-now{% endif %}
+                {% if watch.notification_muted %}notification_muted{% endif %}
                 ">
                 <td class="inline checkbox-uuid" ><input name="uuids"  type="checkbox" value="{{ watch.uuid}} " > <span>{{ loop.index+pagination.skip }}</span></td>
                 <td class="inline watch-controls">
-                    {% if not watch.paused %}
-                    <a class="state-off" href="{{url_for('watchlist.index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause" ></a>
-                    {% else %}
-                    <a class="state-on" href="{{url_for('watchlist.index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
-                    {% endif %}
-                    {% set mute_label = 'UnMute notification' if watch.notification_muted else 'Mute notification' %}
-                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('watchlist.index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="{{ mute_label }}" title="{{ mute_label }}" class="icon icon-mute" ></a>
+                    <a class="ajax-op state-off pause-toggle" data-op="pause" href="{{url_for('watchlist.index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause" ></a>
+                    <a class="ajax-op state-on pause-toggle"  data-op="pause" style="display: none" href="{{url_for('watchlist.index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
+
+                    <a class="ajax-op state-off mute-toggle" data-op="mute" href="{{url_for('watchlist.index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notification" title="Mute notification" class="icon icon-mute" ></a>
+                    <a class="ajax-op state-on mute-toggle" data-op="mute"  style="display: none" href="{{url_for('watchlist.index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="UnMute notification" title="UnMute notification" class="icon icon-mute" ></a>
                 </td>
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
                     <a class="link-spread" href="{{url_for('ui.form_share_put_watch', uuid=watch.uuid)}}"><img src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="status-icon icon icon-spread" title="Create a link to share watch config with others" ></a>
 
                     {% if watch.get_fetch_backend == "html_webdriver"
-                         or (  watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
+                         or ( watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
                          or "extra_browser_" in watch.get_fetch_backend
                     %}
                     <img class="status-icon" src="{{url_for('static_content', group='images', filename='google-chrome-icon.png')}}" alt="Using a Chrome browser" title="Using a Chrome browser" >
                     {% endif %}
 
-                    {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" >{% endif %}
-                    {% if watch.has_browser_steps %}<img class="status-icon status-browsersteps" src="{{url_for('static_content', group='images', filename='steps.svg')}}" title="Browser Steps is enabled" >{% endif %}
-                    {% if watch.last_error is defined and watch.last_error != False %}
-                    <div class="fetch-error">{{ watch.last_error }}
+                    {% if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" alt="Converting PDF to text" >{% endif %}
+                    {% if watch.has_browser_steps %}<img class="status-icon status-browsersteps" src="{{url_for('static_content', group='images', filename='steps.svg')}}" alt="Browser Steps is enabled" >{% endif %}
 
-                        {% if '403' in watch.last_error %}
-                            {% if has_proxies %}
-                                <a href="{{ url_for('settings.settings_page', uuid=watch.uuid) }}#proxies">Try other proxies/location</a>&nbsp;
-                            {% endif %}
-                            <a href="{{ url_for('settings.settings_page', uuid=watch.uuid) }}#proxies">Try adding external proxies/locations</a>
-                        
-                        {% endif %}
-                        {% if 'empty result or contain only an image' in watch.last_error %}
-                            <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Detecting-changes-in-images">more help here</a>.
-                        {% endif %}
-                    </div>
-                    {% endif %}
-                    {% if watch.last_notification_error is defined and watch.last_notification_error != False %}
-                    <div class="fetch-error notification-error"><a href="{{url_for('settings.notification_logs')}}">{{ watch.last_notification_error }}</a></div>
-                    {% endif %}
+                    <div class="error-text" style="display:none;">{{ watch.compile_error_texts(has_proxies=datastore.proxy_list)|safe }}</div>
 
                     {% if watch['processor'] == 'text_json_diff'  %}
                         {% if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data']  %}
@@ -190,12 +173,13 @@
                 </td>
 {% endif %}
             {#last_checked becomes fetch-start-time#}
-                <td class="last-checked" data-timestamp="{{ watch.last_checked }}" {% if checking_now %} data-fetchduration={{ watch.fetch_time }} data-eta_complete="{{ watch.last_checked+watch.fetch_time }}" {% endif %} >
-                    {% if checking_now %}
-                        <span class="spinner"></span><span> Checking now</span>
-                    {% else %}
-                        {{watch|format_last_checked_time|safe}}</td>
-                    {% endif %}
+                <td class="last-checked" data-timestamp="{{ watch.last_checked }}" data-fetchduration={{ watch.fetch_time }} data-eta_complete="{{ watch.last_checked+watch.fetch_time }}" >
+                    <div class="spinner-wrapper" style="display:none;" >
+                        <span class="spinner"></span><span>&nbsp;Checking now</span>
+                    </div>
+                    <span class="innertext">{{watch|format_last_checked_time|safe}}</span>
+                </td>
+
 
                 <td class="last-changed" data-timestamp="{{ watch.last_changed }}">{% if watch.history_n >=2 and watch.last_changed >0 %}
                     {{watch.last_changed|format_timestamp_timeago}}
@@ -204,15 +188,17 @@
                     {% endif %}
                 </td>
                 <td>
-                    <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('ui.form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
-                       class="recheck pure-button pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
+                    <a href="" class="already-in-queue-button recheck pure-button pure-button-primary" style="display: none;" disabled="disabled">Queued</a>
+
+                    <a href="{{ url_for('ui.form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}" data-op='recheck' class="ajax-op recheck pure-button pure-button-primary">Recheck</a>
                     <a href="{{ url_for('ui.ui_edit.edit_page', uuid=watch.uuid, tag=active_tag_uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
+
                     {% if watch.history_n >= 2 %}
 
                         {% set open_diff_in_new_tab = datastore.data['settings']['application']['ui'].get('open_diff_in_new_tab') %}
                         {% set target_attr = ' target="' ~ watch.uuid ~ '"' if open_diff_in_new_tab else '' %}
 
-                        {%  if is_unviewed %}
+                        {% if watch.has_unviewed %}
                            <a href="{{ url_for('ui.ui_views.diff_history_page', uuid=watch.uuid, from_version=watch.get_from_version_based_on_last_viewed) }}" {{target_attr}} class="pure-button pure-button-primary diff-link">History</a>
                         {% else %}
                            <a href="{{ url_for('ui.ui_views.diff_history_page', uuid=watch.uuid)}}" {{target_attr}} class="pure-button pure-button-primary diff-link">History</a>
@@ -229,18 +215,14 @@
             </tbody>
         </table>
         <ul id="post-list-buttons">
-            {% if errored_count %}
-            <li>
-                <a href="{{url_for('watchlist.index', with_errors=1, tag=request.args.get('tag')) }}" class="pure-button button-tag button-error ">With errors ({{ errored_count }})</a>
+            <li id="post-list-with-errors" class="{% if errored_count %}has-error{% endif %}" style="display: none;" >
+                <a href="{{url_for('watchlist.index', with_errors=1, tag=request.args.get('tag')) }}" class="pure-button button-tag button-error">With errors ({{ errored_count }})</a>
             </li>
-            {% endif %}
-            {% if has_unviewed %}
-            <li>
-                <a href="{{url_for('ui.mark_all_viewed',with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Mark all viewed</a>
+            <li id="post-list-mark-views" class="{% if has_unviewed %}has-unviewed{% endif %}" style="display: none;" >
+                <a href="{{url_for('ui.mark_all_viewed',with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag " id="mark-all-viewed">Mark all viewed</a>
             </li>
-            {% endif %}
             <li>
-               <a href="{{ url_for('ui.form_watch_checknow', tag=active_tag_uuid, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Recheck
+               <a href="{{ url_for('ui.form_watch_checknow', tag=active_tag_uuid, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag" id="recheck-all">Recheck
                 all {% if active_tag_uuid %} in "{{active_tag.title}}"{%endif%}</a>
             </li>
             <li>
@@ -251,4 +233,4 @@
     </div>
     </form>
 </div>
-{% endblock %}
+{% endblock %}

changedetectionio/conditions/__init__.py

@@ -5,7 +5,7 @@ from json_logic.builtins import BUILTINS
 from .exceptions import EmptyConditionRuleRowNotUsable
 from .pluggy_interface import plugin_manager  # Import the pluggy plugin manager
 from . import default_plugin
-
+from loguru import logger
 # List of all supported JSON Logic operators
 operator_choices = [
     (None, "Choose one - Operator"),
@@ -102,12 +102,33 @@ def execute_ruleset_against_all_plugins(current_watch_uuid: str, application_dat
         if complete_rules:
             # Give all plugins a chance to update the data dict again (that we will test the conditions against)
             for plugin in plugin_manager.get_plugins():
-                new_execute_data = plugin.add_data(current_watch_uuid=current_watch_uuid,
-                                                   application_datastruct=application_datastruct,
-                                                   ephemeral_data=ephemeral_data)
+                try:
+                    import concurrent.futures
+                    import time
+                    
+                    with concurrent.futures.ThreadPoolExecutor() as executor:
+                        future = executor.submit(
+                            plugin.add_data,
+                            current_watch_uuid=current_watch_uuid,
+                            application_datastruct=application_datastruct,
+                            ephemeral_data=ephemeral_data
+                        )
+                        logger.debug(f"Trying plugin {plugin}....")
 
-                if new_execute_data and isinstance(new_execute_data, dict):
-                    EXECUTE_DATA.update(new_execute_data)
+                        # Set a timeout of 10 seconds
+                        try:
+                            new_execute_data = future.result(timeout=10)
+                            if new_execute_data and isinstance(new_execute_data, dict):
+                                EXECUTE_DATA.update(new_execute_data)
+
+                        except concurrent.futures.TimeoutError:
+                            # The plugin took too long, abort processing for this watch
+                            raise Exception(f"Plugin {plugin.__class__.__name__} took more than 10 seconds to run.")
+                except Exception as e:
+                    # Log the error but continue with the next plugin
+                    import logging
+                    logging.error(f"Error executing plugin {plugin.__class__.__name__}: {str(e)}")
+                    continue
 
             # Create the ruleset
             ruleset = convert_to_jsonlogic(logic_operator=logic_operator, rule_dict=complete_rules)
@@ -132,3 +153,18 @@ for plugin in plugin_manager.get_plugins():
     if isinstance(new_field_choices, list):
         field_choices.extend(new_field_choices)
 
+def collect_ui_edit_stats_extras(watch):
+    """Collect and combine HTML content from all plugins that implement ui_edit_stats_extras"""
+    extras_content = []
+    
+    for plugin in plugin_manager.get_plugins():
+        try:
+            content = plugin.ui_edit_stats_extras(watch=watch)
+            if content:
+                extras_content.append(content)
+        except Exception as e:
+            # Skip plugins that don't implement the hook or have errors
+            pass
+            
+    return "\n".join(extras_content) if extras_content else ""
+

changedetectionio/conditions/pluggy_interface.py

@@ -1,5 +1,8 @@
 import pluggy
-from . import default_plugin  # Import the default plugin
+import os
+import importlib
+import sys
+from . import default_plugin
 
 # ✅ Ensure that the namespace in HookspecMarker matches PluginManager
 PLUGIN_NAMESPACE = "changedetectionio_conditions"
@@ -30,6 +33,11 @@ class ConditionsSpec:
     def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
         """Add to the datadict"""
         pass
+        
+    @hookspec
+    def ui_edit_stats_extras(watch):
+        """Return HTML content to add to the stats tab in the edit view"""
+        pass
 
 # ✅ Set up Pluggy Plugin Manager
 plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE)
@@ -40,5 +48,27 @@ plugin_manager.add_hookspecs(ConditionsSpec)
 # ✅ Register built-in plugins manually
 plugin_manager.register(default_plugin, "default_plugin")
 
+# ✅ Load plugins from the plugins directory
+def load_plugins_from_directory():
+    plugins_dir = os.path.join(os.path.dirname(__file__), 'plugins')
+    if not os.path.exists(plugins_dir):
+        return
+        
+    # Get all Python files (excluding __init__.py)
+    for filename in os.listdir(plugins_dir):
+        if filename.endswith(".py") and filename != "__init__.py":
+            module_name = filename[:-3]  # Remove .py extension
+            module_path = f"changedetectionio.conditions.plugins.{module_name}"
+            
+            try:
+                module = importlib.import_module(module_path)
+                # Register the plugin with pluggy
+                plugin_manager.register(module, module_name)
+            except (ImportError, AttributeError) as e:
+                print(f"Error loading plugin {module_name}: {e}")
+
+# Load plugins from the plugins directory
+load_plugins_from_directory()
+
 # ✅ Discover installed plugins from external packages (if any)
 plugin_manager.load_setuptools_entrypoints(PLUGIN_NAMESPACE)

changedetectionio/conditions/plugins/__init__.py

@@ -0,0 +1 @@
+# Import plugins package to make them discoverable

changedetectionio/conditions/plugins/levenshtein_plugin.py

@@ -0,0 +1,107 @@
+import pluggy
+from loguru import logger
+
+# Support both plugin systems
+conditions_hookimpl = pluggy.HookimplMarker("changedetectionio_conditions")
+global_hookimpl = pluggy.HookimplMarker("changedetectionio")
+
+def levenshtein_ratio_recent_history(watch, incoming_text=None):
+    try:
+        from Levenshtein import ratio, distance
+        k = list(watch.history.keys())
+        a = None
+        b = None
+
+        # When called from ui_edit_stats_extras, we don't have incoming_text
+        if incoming_text is None:
+            a = watch.get_history_snapshot(timestamp=k[-1])  # Latest snapshot
+            b = watch.get_history_snapshot(timestamp=k[-2])  # Previous snapshot
+
+        # Needs atleast one snapshot
+        elif len(k) >= 1: # Should be atleast one snapshot to compare against
+            a = watch.get_history_snapshot(timestamp=k[-1]) # Latest saved snapshot
+            b = incoming_text if incoming_text else k[-2]
+
+        if a and b:
+            distance_value = distance(a, b)
+            ratio_value = ratio(a, b)
+            return {
+                'distance': distance_value,
+                'ratio': ratio_value,
+                'percent_similar': round(ratio_value * 100, 2)
+            }
+    except Exception as e:
+        logger.warning(f"Unable to calc similarity: {str(e)}")
+
+    return ''
+
+@conditions_hookimpl
+def register_operators():
+    pass
+
+@conditions_hookimpl
+def register_operator_choices():
+    pass
+
+
+@conditions_hookimpl
+def register_field_choices():
+    return [
+        ("levenshtein_ratio", "Levenshtein - Text similarity ratio"),
+        ("levenshtein_distance", "Levenshtein - Text change distance"),
+    ]
+
+@conditions_hookimpl
+def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
+    res = {}
+    watch = application_datastruct['watching'].get(current_watch_uuid)
+    # ephemeral_data['text'] will be the current text after filters, they may have edited filters but not saved them yet etc
+
+    if watch and 'text' in ephemeral_data:
+        lev_data = levenshtein_ratio_recent_history(watch, ephemeral_data.get('text',''))
+        if isinstance(lev_data, dict):
+            res['levenshtein_ratio'] = lev_data.get('ratio', 0)
+            res['levenshtein_similarity'] = lev_data.get('percent_similar', 0)
+            res['levenshtein_distance'] = lev_data.get('distance', 0)
+
+    return res
+
+@global_hookimpl
+def ui_edit_stats_extras(watch):
+    """Add Levenshtein stats to the UI using the global plugin system"""
+    """Generate the HTML for Levenshtein stats - shared by both plugin systems"""
+    if len(watch.history.keys()) < 2:
+        return "<p>Not enough history to calculate Levenshtein metrics</p>"
+    
+    try:
+        lev_data = levenshtein_ratio_recent_history(watch)
+        if not lev_data or not isinstance(lev_data, dict):
+            return "<p>Unable to calculate Levenshtein metrics</p>"
+            
+        html = f"""
+        <div class="levenshtein-stats">
+            <h4>Levenshtein Text Similarity Details</h4>
+            <table class="pure-table">
+                <tbody>
+                    <tr>
+                        <td>Raw distance (edits needed)</td>
+                        <td>{lev_data['distance']}</td>
+                    </tr>
+                    <tr>
+                        <td>Similarity ratio</td>
+                        <td>{lev_data['ratio']:.4f}</td>
+                    </tr>
+                    <tr>
+                        <td>Percent similar</td>
+                        <td>{lev_data['percent_similar']}%</td>
+                    </tr>
+                </tbody>
+            </table>
+            <p style="font-size: 80%;">Levenshtein metrics compare the last two snapshots, measuring how many character edits are needed to transform one into the other.</p>
+        </div>
+        """
+        return html
+    except Exception as e:
+        logger.error(f"Error generating Levenshtein UI extras: {str(e)}")
+        return "<p>Error calculating Levenshtein metrics</p>"
+        

changedetectionio/conditions/plugins/wordcount_plugin.py

@@ -0,0 +1,82 @@
+import pluggy
+from loguru import logger
+
+# Support both plugin systems
+conditions_hookimpl = pluggy.HookimplMarker("changedetectionio_conditions")
+global_hookimpl = pluggy.HookimplMarker("changedetectionio")
+
+def count_words_in_history(watch, incoming_text=None):
+    """Count words in snapshot text"""
+    try:
+        if incoming_text is not None:
+            # When called from add_data with incoming text
+            return len(incoming_text.split())
+        elif watch.history.keys():
+            # When called from UI extras to count latest snapshot
+            latest_key = list(watch.history.keys())[-1]
+            latest_content = watch.get_history_snapshot(latest_key)
+            return len(latest_content.split())
+        return 0
+    except Exception as e:
+        logger.error(f"Error counting words: {str(e)}")
+        return 0
+
+# Implement condition plugin hooks
+@conditions_hookimpl
+def register_operators():
+    # No custom operators needed
+    return {}
+
+@conditions_hookimpl
+def register_operator_choices():
+    # No custom operator choices needed
+    return []
+
+@conditions_hookimpl
+def register_field_choices():
+    # Add a field that will be available in conditions
+    return [
+        ("word_count", "Word count of content"),
+    ]
+
+@conditions_hookimpl
+def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
+    """Add word count data for conditions"""
+    result = {}
+    watch = application_datastruct['watching'].get(current_watch_uuid)
+    
+    if watch and 'text' in ephemeral_data:
+        word_count = count_words_in_history(watch, ephemeral_data['text'])
+        result['word_count'] = word_count
+    
+    return result
+
+def _generate_stats_html(watch):
+    """Generate the HTML content for the stats tab"""
+    word_count = count_words_in_history(watch)
+    
+    html = f"""
+    <div class="word-count-stats">
+        <h4>Content Analysis</h4>
+        <table class="pure-table">
+            <tbody>
+                <tr>
+                    <td>Word count (latest snapshot)</td>
+                    <td>{word_count}</td>
+                </tr>
+            </tbody>
+        </table>
+        <p style="font-size: 80%;">Word count is a simple measure of content length, calculated by splitting text on whitespace.</p>
+    </div>
+    """
+    return html
+
+@conditions_hookimpl
+def ui_edit_stats_extras(watch):
+    """Add word count stats to the UI through conditions plugin system"""
+    return _generate_stats_html(watch)
+
+@global_hookimpl
+def ui_edit_stats_extras(watch):
+    """Add word count stats to the UI using the global plugin system"""
+    return _generate_stats_html(watch)

changedetectionio/content_fetchers/playwright.py

@@ -186,7 +186,7 @@ class fetcher(Fetcher):
             self.page = context.new_page()
 
             # Listen for all console events and handle errors
-            self.page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
+            self.page.on("console", lambda msg: logger.debug(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
 
             # Re-use as much code from browser steps as possible so its the same
             from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
@@ -194,7 +194,6 @@ class fetcher(Fetcher):
             browsersteps_interface.page = self.page
 
             response = browsersteps_interface.action_goto_url(value=url)
-            self.headers = response.all_headers()
 
             if response is None:
                 context.close()
@@ -202,6 +201,8 @@ class fetcher(Fetcher):
                 logger.debug("Content Fetcher > Response object from the browser communication was none")
                 raise EmptyReply(url=url, status_code=None)
 
+            self.headers = response.all_headers()
+
             try:
                 if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
                     browsersteps_interface.action_execute_js(value=self.webdriver_js_execute_code, selector=None)

changedetectionio/content_fetchers/puppeteer.py

@@ -147,7 +147,7 @@ class fetcher(Fetcher):
                          is_binary,
                          empty_pages_are_a_change
                          ):
-
+        import re
         self.delete_browser_steps_screenshots()
         extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
 
@@ -172,6 +172,17 @@ class fetcher(Fetcher):
         # headless - ask a new page
         self.page = (pages := await browser.pages) and len(pages) or await browser.newPage()
 
+        if '--window-size' in self.browser_connection_url:
+            # Be sure the viewport is always the window-size, this is often not the same thing
+            match = re.search(r'--window-size=(\d+),(\d+)', self.browser_connection_url)
+            if match:
+                logger.debug(f"Setting viewport to same as --window-size in browser connection URL {int(match.group(1))},{int(match.group(2))}")
+                await self.page.setViewport({
+                    "width": int(match.group(1)),
+                    "height": int(match.group(2))
+                })
+                logger.debug(f"Puppeteer viewport size {self.page.viewport}")
+
         try:
             from pyppeteerstealth import inject_evasions_into_page
         except ImportError:
@@ -218,7 +229,6 @@ class fetcher(Fetcher):
 
         response = await self.page.goto(url, waitUntil="load")
 
-
         if response is None:
             await self.page.close()
             await browser.close()

changedetectionio/content_fetchers/requests.py

@@ -28,6 +28,7 @@ class fetcher(Fetcher):
 
         import chardet
         import requests
+        from requests.exceptions import ProxyError, ConnectionError, RequestException
 
         if self.browser_steps_get_valid_steps():
             raise BrowserStepsInUnsupportedFetcher(url=url)
@@ -52,14 +53,19 @@ class fetcher(Fetcher):
         if strtobool(os.getenv('ALLOW_FILE_URI', 'false')) and url.startswith('file://'):
             from requests_file import FileAdapter
             session.mount('file://', FileAdapter())
-
-        r = session.request(method=request_method,
-                            data=request_body.encode('utf-8') if type(request_body) is str else request_body,
-                            url=url,
-                            headers=request_headers,
-                            timeout=timeout,
-                            proxies=proxies,
-                            verify=False)
+        try:
+            r = session.request(method=request_method,
+                                data=request_body.encode('utf-8') if type(request_body) is str else request_body,
+                                url=url,
+                                headers=request_headers,
+                                timeout=timeout,
+                                proxies=proxies,
+                                verify=False)
+        except Exception as e:
+            msg = str(e)
+            if proxies and 'SOCKSHTTPSConnectionPool' in msg:
+                msg = f"Proxy connection failed? {msg}"
+            raise Exception(msg) from e
 
         # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
         # For example - some sites don't tell us it's utf-8, but return utf-8 content

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -10,6 +10,7 @@ async () => {
             'article épuisé',
             'artikel zurzeit vergriffen',
             'as soon as stock is available',
+            'aucune offre n\'est disponible',
             'ausverkauft', // sold out
             'available for back order',
             'awaiting stock',
@@ -25,9 +26,8 @@ async () => {
             'dieser artikel ist bald wieder verfügbar',
             'dostępne wkrótce',
             'en rupture',
-            'en rupture de stock',
-            'épuisé',
             'esgotado',
+            'in kürze lieferbar',
             'indisponible',
             'indisponível',
             'isn\'t in stock right now',
@@ -50,10 +50,12 @@ async () => {
             'niet leverbaar',
             'niet op voorraad',
             'no disponible',
-            'non disponibile',
-            'non disponible',
+            'no featured offers available',
+            'no longer available',
             'no longer in stock',
             'no tickets available',
+            'non disponibile',
+            'non disponible',
             'not available',
             'not currently available',
             'not in stock',
@@ -89,13 +91,15 @@ async () => {
             'vergriffen',
             'vorbestellen',
             'vorbestellung ist bald möglich',
-            'we don\'t currently have any',
             'we couldn\'t find any products that match',
             'we do not currently have an estimate of when this product will be back in stock.',
+            'we don\'t currently have any',
             'we don\'t know when or if this item will be back in stock.',
             'we were not able to find a match',
             'when this arrives in stock',
+            'when this item is available to order',
             'zur zeit nicht an lager',
+            'épuisé',
             '品切れ',
             '已售',
             '已售完',
@@ -122,6 +126,20 @@ async () => {
         // so it's good to filter to just the 'above the fold' elements
         // and it should be atleast 100px from the top to ignore items in the toolbar, sometimes menu items like "Coming soon" exist
 
+        function elementIsInEyeBallRange(element) {
+            // outside the 'fold' or some weird text in the heading area
+            // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
+            // Note: theres also an automated test that places the 'out of stock' text fairly low down
+            // Skip text that could be in the header area
+            if (element.getBoundingClientRect().bottom + window.scrollY <= 300 ) {
+                return false;
+            }
+            // Skip text that could be much further down (like a list of "you may like" products that have 'sold out' in there
+            if (element.getBoundingClientRect().bottom + window.scrollY >= 1300 ) {
+                return false;
+            }
+            return true;
+        }
 
 // @todo - if it's SVG or IMG, go into image diff mode
 
@@ -158,9 +176,7 @@ async () => {
         for (let i = elementsToScan.length - 1; i >= 0; i--) {
             const element = elementsToScan[i];
 
-            // outside the 'fold' or some weird text in the heading area
-            // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
-            if (element.getBoundingClientRect().top + window.scrollY >= vh || element.getBoundingClientRect().top + window.scrollY <= 100) {
+            if (!elementIsInEyeBallRange(element)) {
                 continue
             }
 
@@ -174,11 +190,11 @@ async () => {
             } catch (e) {
                 console.warn('stock-not-in-stock.js scraper - handling element for gettext failed', e);
             }
-
             if (elementText.length) {
                 // try which ones could mean its in stock
                 if (negateOutOfStockRegex.test(elementText) && !elementText.includes('(0 products)')) {
                     console.log(`Negating/overriding 'Out of Stock' back to "Possibly in stock" found "${elementText}"`)
+                    element.style.border = "2px solid green"; // highlight the element that was detected as in stock
                     return 'Possibly in stock';
                 }
             }
@@ -187,10 +203,8 @@ async () => {
         // OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK
         for (let i = elementsToScan.length - 1; i >= 0; i--) {
             const element = elementsToScan[i];
-            // outside the 'fold' or some weird text in the heading area
-            // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
-            // Note: theres also an automated test that places the 'out of stock' text fairly low down
-            if (element.getBoundingClientRect().top + window.scrollY >= vh + 250 || element.getBoundingClientRect().top + window.scrollY <= 100) {
+
+            if (!elementIsInEyeBallRange(element)) {
                 continue
             }
             elementText = "";
@@ -205,6 +219,7 @@ async () => {
                 for (const outOfStockText of outOfStockTexts) {
                     if (elementText.includes(outOfStockText)) {
                         console.log(`Selected 'Out of Stock' - found text "${outOfStockText}" - "${elementText}" - offset top ${element.getBoundingClientRect().top}, page height is ${vh}`)
+                        element.style.border = "2px solid red"; // highlight the element that was detected as out of stock
                         return outOfStockText; // item is out of stock
                     }
                 }

changedetectionio/content_fetchers/res/xpath_element_scraper.js

@@ -202,7 +202,6 @@ async (options) => {
         // Foreach filter, go and find it on the page and add it to the results so we can visualise it again
         for (const f of include_filters) {
             bbox = false;
-            q = false;
 
             if (!f.length) {
                 console.log("xpath_element_scraper: Empty filter, skipping");
@@ -255,7 +254,7 @@ async (options) => {
                             console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
                         } catch (e) {
                             console.log(e)
-                            console.log("xpath_element_scraper: error looking up q.ownerElement")
+                            console.log("xpath_element_scraper: error looking up node.ownerElement")
                         }
                     }
 

changedetectionio/content_fetchers/screenshot_handler.py

@@ -31,33 +31,33 @@ def stitch_images_worker(pipe_conn, chunks_bytes, original_page_height, capture_
         # Draw caption on top (overlaid, not extending canvas)
         draw = ImageDraw.Draw(stitched)
 
-
-        caption_text = f"WARNING: Screenshot was {original_page_height}px but trimmed to {capture_height}px because it was too long"
-        padding = 10
-        font_size = 35
-        font_color = (255, 0, 0)
-        background_color = (255, 255, 255)
+        if original_page_height > capture_height:
+            caption_text = f"WARNING: Screenshot was {original_page_height}px but trimmed to {capture_height}px because it was too long"
+            padding = 10
+            font_size = 35
+            font_color = (255, 0, 0)
+            background_color = (255, 255, 255)
 
 
-        # Try to load a proper font
-        try:
-            font = ImageFont.truetype("arial.ttf", font_size)
-        except IOError:
-            font = ImageFont.load_default()
+            # Try to load a proper font
+            try:
+                font = ImageFont.truetype("arial.ttf", font_size)
+            except IOError:
+                font = ImageFont.load_default()
 
-        bbox = draw.textbbox((0, 0), caption_text, font=font)
-        text_width = bbox[2] - bbox[0]
-        text_height = bbox[3] - bbox[1]
+            bbox = draw.textbbox((0, 0), caption_text, font=font)
+            text_width = bbox[2] - bbox[0]
+            text_height = bbox[3] - bbox[1]
 
-        # Draw white rectangle background behind text
-        rect_top = 0
-        rect_bottom = text_height + 2 * padding
-        draw.rectangle([(0, rect_top), (max_width, rect_bottom)], fill=background_color)
+            # Draw white rectangle background behind text
+            rect_top = 0
+            rect_bottom = text_height + 2 * padding
+            draw.rectangle([(0, rect_top), (max_width, rect_bottom)], fill=background_color)
 
-        # Draw text centered horizontally, 10px padding from top of the rectangle
-        text_x = (max_width - text_width) // 2
-        text_y = padding
-        draw.text((text_x, text_y), caption_text, font=font, fill=font_color)
+            # Draw text centered horizontally, 10px padding from top of the rectangle
+            text_x = (max_width - text_width) // 2
+            text_y = padding
+            draw.text((text_x, text_y), caption_text, font=font, fill=font_color)
 
         # Encode and send image
         output = io.BytesIO()

changedetectionio/content_fetchers/webdriver_selenium.py

@@ -10,16 +10,13 @@ class fetcher(Fetcher):
     else:
         fetcher_description = "WebDriver Chrome/Javascript"
 
-    # Configs for Proxy setup
-    # In the ENV vars, is prefixed with "webdriver_", so it is for example "webdriver_sslProxy"
-    selenium_proxy_settings_mappings = ['proxyType', 'ftpProxy', 'httpProxy', 'noProxy',
-                                        'proxyAutoconfigUrl', 'sslProxy', 'autodetect',
-                                        'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
     proxy = None
+    proxy_url = None
 
     def __init__(self, proxy_override=None, custom_browser_connection_url=None):
         super().__init__()
-        from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
+        from urllib.parse import urlparse
+        from selenium.webdriver.common.proxy import Proxy
 
         # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
         if not custom_browser_connection_url:
@@ -28,25 +25,27 @@ class fetcher(Fetcher):
             self.browser_connection_is_custom = True
             self.browser_connection_url = custom_browser_connection_url
 
-        # If any proxy settings are enabled, then we should setup the proxy object
-        proxy_args = {}
-        for k in self.selenium_proxy_settings_mappings:
-            v = os.getenv('webdriver_' + k, False)
-            if v:
-                proxy_args[k] = v.strip('"')
 
-        # Map back standard HTTP_ and HTTPS_PROXY to webDriver httpProxy/sslProxy
-        if not proxy_args.get('webdriver_httpProxy') and self.system_http_proxy:
-            proxy_args['httpProxy'] = self.system_http_proxy
-        if not proxy_args.get('webdriver_sslProxy') and self.system_https_proxy:
-            proxy_args['httpsProxy'] = self.system_https_proxy
+        ##### PROXY SETUP #####
 
-        # Allows override the proxy on a per-request basis
-        if proxy_override is not None:
-            proxy_args['httpProxy'] = proxy_override
+        proxy_sources = [
+            self.system_http_proxy,
+            self.system_https_proxy,
+            os.getenv('webdriver_proxySocks'),
+            os.getenv('webdriver_socksProxy'),
+            os.getenv('webdriver_proxyHttp'),
+            os.getenv('webdriver_httpProxy'),
+            os.getenv('webdriver_proxyHttps'),
+            os.getenv('webdriver_httpsProxy'),
+            os.getenv('webdriver_sslProxy'),
+            proxy_override, # last one should override
+        ]
+        # The built in selenium proxy handling is super unreliable!!! so we just grab which ever proxy setting we can find and throw it in --proxy-server=
+        for k in filter(None, proxy_sources):
+            if not k:
+                continue
+            self.proxy_url = k.strip()
 
-        if proxy_args:
-            self.proxy = SeleniumProxy(raw=proxy_args)
 
     def run(self,
             url,
@@ -59,9 +58,7 @@ class fetcher(Fetcher):
             is_binary=False,
             empty_pages_are_a_change=False):
 
-        from selenium import webdriver
         from selenium.webdriver.chrome.options import Options as ChromeOptions
-        from selenium.common.exceptions import WebDriverException
         # request_body, request_method unused for now, until some magic in the future happens.
 
         options = ChromeOptions()
@@ -76,59 +73,62 @@ class fetcher(Fetcher):
         for opt in CHROME_OPTIONS:
             options.add_argument(opt)
 
-        if self.proxy:
-            options.proxy = self.proxy
+        # 1. proxy_config /Proxy(proxy_config) selenium object is REALLY unreliable
+        # 2. selenium-wire cant be used because the websocket version conflicts with pypeteer-ng
+        # 3. selenium only allows ONE runner at a time by default!
+        # 4. driver must use quit() or it will continue to block/hold the selenium process!!
 
-        self.driver = webdriver.Remote(
-            command_executor=self.browser_connection_url,
-            options=options)
+        if self.proxy_url:
+            options.add_argument(f'--proxy-server={self.proxy_url}')
+
+        from selenium.webdriver.remote.remote_connection import RemoteConnection
+        from selenium.webdriver.remote.webdriver import WebDriver as RemoteWebDriver
+        driver = None
+        try:
+            # Create the RemoteConnection and set timeout (e.g., 30 seconds)
+            remote_connection = RemoteConnection(
+                self.browser_connection_url,
+            )
+            remote_connection.set_timeout(30)  # seconds
+
+            # Now create the driver with the RemoteConnection
+            driver = RemoteWebDriver(
+                command_executor=remote_connection,
+                options=options
+            )
+
+            driver.set_page_load_timeout(int(os.getenv("WEBDRIVER_PAGELOAD_TIMEOUT", 45)))
+        except Exception as e:
+            if driver:
+                driver.quit()
+            raise e
 
         try:
-            self.driver.get(url)
-        except WebDriverException as e:
-            # Be sure we close the session window
-            self.quit()
-            raise
+            driver.get(url)
 
-        if not "--window-size" in os.getenv("CHROME_OPTIONS", ""):
-            self.driver.set_window_size(1280, 1024)
+            if not "--window-size" in os.getenv("CHROME_OPTIONS", ""):
+                driver.set_window_size(1280, 1024)
 
-        self.driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+            driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
 
-        if self.webdriver_js_execute_code is not None:
-            self.driver.execute_script(self.webdriver_js_execute_code)
-            # Selenium doesn't automatically wait for actions as good as Playwright, so wait again
-            self.driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+            if self.webdriver_js_execute_code is not None:
+                driver.execute_script(self.webdriver_js_execute_code)
+                # Selenium doesn't automatically wait for actions as good as Playwright, so wait again
+                driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
 
+            # @todo - how to check this? is it possible?
+            self.status_code = 200
+            # @todo somehow we should try to get this working for WebDriver
+            # raise EmptyReply(url=url, status_code=r.status_code)
 
-        # @todo - how to check this? is it possible?
-        self.status_code = 200
-        # @todo somehow we should try to get this working for WebDriver
-        # raise EmptyReply(url=url, status_code=r.status_code)
+            # @todo - dom wait loaded?
+            time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay)
+            self.content = driver.page_source
+            self.headers = {}
+            self.screenshot = driver.get_screenshot_as_png()
+        except Exception as e:
+            driver.quit()
+            raise e
 
-        # @todo - dom wait loaded?
-        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay)
-        self.content = self.driver.page_source
-        self.headers = {}
+        driver.quit()
 
-        self.screenshot = self.driver.get_screenshot_as_png()
-
-    # Does the connection to the webdriver work? run a test connection.
-    def is_ready(self):
-        from selenium import webdriver
-        from selenium.webdriver.chrome.options import Options as ChromeOptions
-
-        self.driver = webdriver.Remote(
-            command_executor=self.command_executor,
-            options=ChromeOptions())
-
-        # driver.quit() seems to cause better exceptions
-        self.quit()
-        return True
-
-    def quit(self, watch=None):
-        if self.driver:
-            try:
-                self.driver.quit()
-            except Exception as e:
-                logger.debug(f"Content Fetcher > Exception in chrome shutdown/quit {str(e)}")

changedetectionio/custom_queue.py

@@ -0,0 +1,52 @@
+import queue
+from blinker import signal
+from loguru import logger
+
+class SignalPriorityQueue(queue.PriorityQueue):
+    """
+    Extended PriorityQueue that sends a signal when items with a UUID are added.
+    
+    This class extends the standard PriorityQueue and adds a signal emission
+    after an item is put into the queue. If the item contains a UUID, the signal
+    is sent with that UUID as a parameter.
+    """
+    
+    def __init__(self, maxsize=0):
+        super().__init__(maxsize)
+        try:
+            self.queue_length_signal = signal('queue_length')
+        except Exception as e:
+            logger.critical(f"Exception: {e}")
+
+    def put(self, item, block=True, timeout=None):
+        # Call the parent's put method first
+        super().put(item, block, timeout)
+        
+        # After putting the item in the queue, check if it has a UUID and emit signal
+        if hasattr(item, 'item') and isinstance(item.item, dict) and 'uuid' in item.item:
+            uuid = item.item['uuid']
+            # Get the signal and send it if it exists
+            watch_check_update = signal('watch_check_update')
+            if watch_check_update:
+                # Send the watch_uuid parameter
+                watch_check_update.send(watch_uuid=uuid)
+        
+        # Send queue_length signal with current queue size
+        try:
+
+            if self.queue_length_signal:
+                self.queue_length_signal.send(length=self.qsize())
+        except Exception as e:
+            logger.critical(f"Exception: {e}")
+
+    def get(self, block=True, timeout=None):
+        # Call the parent's get method first
+        item = super().get(block, timeout)
+        
+        # Send queue_length signal with current queue size
+        try:
+            if self.queue_length_signal:
+                self.queue_length_signal.send(length=self.qsize())
+        except Exception as e:
+            logger.critical(f"Exception: {e}")
+        return item

changedetectionio/flask_app.py

@@ -7,9 +7,11 @@ import queue
 import threading
 import time
 import timeago
+from blinker import signal
 
 from changedetectionio.strtobool import strtobool
 from threading import Event
+from changedetectionio.custom_queue import SignalPriorityQueue
 
 from flask import (
     Flask,
@@ -25,9 +27,12 @@ from flask import (
 )
 from flask_compress import Compress as FlaskCompress
 from flask_login import current_user
-from flask_paginate import Pagination, get_page_parameter
 from flask_restful import abort, Api
 from flask_cors import CORS
+
+# Create specific signals for application events
+# Make this a global singleton to avoid multiple signal objects
+watch_check_update = signal('watch_check_update', doc='Signal sent when a watch check is completed')
 from flask_wtf import CSRFProtect
 from loguru import logger
 
@@ -45,7 +50,7 @@ ticker_thread = None
 
 extra_stylesheets = []
 
-update_q = queue.PriorityQueue()
+update_q = SignalPriorityQueue()
 notification_q = queue.Queue()
 MAX_QUEUE_SIZE = 2000
 
@@ -54,6 +59,9 @@ app = Flask(__name__,
             static_folder="static",
             template_folder="templates")
 
+# Will be initialized in changedetection_app
+socketio_server = None
+
 # Enable CORS, especially useful for the Chrome extension to operate from anywhere
 CORS(app)
 
@@ -115,6 +123,18 @@ def get_darkmode_state():
 def get_css_version():
     return __version__
 
+@app.template_global()
+def get_socketio_path():
+    """Generate the correct Socket.IO path prefix for the client"""
+    # If behind a proxy with a sub-path, we need to respect that path
+    prefix = ""
+    if os.getenv('USE_X_SETTINGS') and 'X-Forwarded-Prefix' in request.headers:
+        prefix = request.headers['X-Forwarded-Prefix']
+
+    # Socket.IO will be available at {prefix}/socket.io/
+    return prefix
+
+
 @app.template_filter('format_number_locale')
 def _jinja2_filter_format_number_locale(value: float) -> str:
     "Formats for example 4000.10 to the local locale default of 4,000.10"
@@ -215,12 +235,15 @@ class User(flask_login.UserMixin):
 def changedetection_app(config=None, datastore_o=None):
     logger.trace("TRACE log is enabled")
 
-    global datastore
+    global datastore, socketio_server
     datastore = datastore_o
 
     # so far just for read-only via tests, but this will be moved eventually to be the main source
     # (instead of the global var)
     app.config['DATASTORE'] = datastore_o
+    
+    # Store the signal in the app config to ensure it's accessible everywhere
+    app.config['watch_check_update_SIGNAL'] = watch_check_update
 
     login_manager = flask_login.LoginManager(app)
     login_manager.login_view = 'login'
@@ -248,6 +271,9 @@ def changedetection_app(config=None, datastore_o=None):
             # RSS access with token is allowed
             elif request.endpoint and 'rss.feed' in request.endpoint:
                 return None
+            # Socket.IO routes - need separate handling
+            elif request.path.startswith('/socket.io/'):
+                return None
             # API routes - use their own auth mechanism (@auth.check_token)
             elif request.path.startswith('/api/'):
                 return None
@@ -444,11 +470,17 @@ def changedetection_app(config=None, datastore_o=None):
 
     # watchlist UI buttons etc
     import changedetectionio.blueprint.ui as ui
-    app.register_blueprint(ui.construct_blueprint(datastore, update_q, running_update_threads, queuedWatchMetaData))
+    app.register_blueprint(ui.construct_blueprint(datastore, update_q, running_update_threads, queuedWatchMetaData, watch_check_update))
 
     import changedetectionio.blueprint.watchlist as watchlist
     app.register_blueprint(watchlist.construct_blueprint(datastore=datastore, update_q=update_q, queuedWatchMetaData=queuedWatchMetaData), url_prefix='')
-    
+
+    # Initialize Socket.IO server
+    from changedetectionio.realtime.socket_server import init_socketio
+    global socketio_server
+    socketio_server = init_socketio(app, datastore)
+    logger.info("Socket.IO server initialized")
+
     # Memory cleanup endpoint
     @app.route('/gc-cleanup', methods=['GET'])
     @login_optionally_required
@@ -467,6 +499,8 @@ def changedetection_app(config=None, datastore_o=None):
     if not os.getenv("GITHUB_REF", False) and not strtobool(os.getenv('DISABLE_VERSION_CHECK', 'no')):
         threading.Thread(target=check_for_new_version).start()
 
+    # Return the Flask app - the Socket.IO will be attached to it but initialized separately
+    # This avoids circular dependencies
     return app
 
 
@@ -502,48 +536,54 @@ def notification_runner():
     global notification_debug_log
     from datetime import datetime
     import json
-    while not app.config.exit.is_set():
-        try:
-            # At the moment only one thread runs (single runner)
-            n_object = notification_q.get(block=False)
-        except queue.Empty:
-            time.sleep(1)
-
-        else:
-
-            now = datetime.now()
-            sent_obj = None
-
+    with app.app_context():
+        while not app.config.exit.is_set():
             try:
-                from changedetectionio.notification.handler import process_notification
+                # At the moment only one thread runs (single runner)
+                n_object = notification_q.get(block=False)
+            except queue.Empty:
+                time.sleep(1)
 
-                # Fallback to system config if not set
-                if not n_object.get('notification_body') and datastore.data['settings']['application'].get('notification_body'):
-                    n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
+            else:
 
-                if not n_object.get('notification_title') and datastore.data['settings']['application'].get('notification_title'):
-                    n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
+                now = datetime.now()
+                sent_obj = None
 
-                if not n_object.get('notification_format') and datastore.data['settings']['application'].get('notification_format'):
-                    n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
-                if n_object.get('notification_urls', {}):
-                    sent_obj = process_notification(n_object, datastore)
+                try:
+                    from changedetectionio.notification.handler import process_notification
 
-            except Exception as e:
-                logger.error(f"Watch URL: {n_object['watch_url']}  Error {str(e)}")
+                    # Fallback to system config if not set
+                    if not n_object.get('notification_body') and datastore.data['settings']['application'].get('notification_body'):
+                        n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
 
-                # UUID wont be present when we submit a 'test' from the global settings
-                if 'uuid' in n_object:
-                    datastore.update_watch(uuid=n_object['uuid'],
-                                           update_obj={'last_notification_error': "Notification error detected, goto notification log."})
+                    if not n_object.get('notification_title') and datastore.data['settings']['application'].get('notification_title'):
+                        n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
+
+                    if not n_object.get('notification_format') and datastore.data['settings']['application'].get('notification_format'):
+                        n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
+                    if n_object.get('notification_urls', {}):
+                        sent_obj = process_notification(n_object, datastore)
+
+                except Exception as e:
+                    logger.error(f"Watch URL: {n_object['watch_url']}  Error {str(e)}")
+
+                    # UUID wont be present when we submit a 'test' from the global settings
+                    if 'uuid' in n_object:
+                        datastore.update_watch(uuid=n_object['uuid'],
+                                               update_obj={'last_notification_error': "Notification error detected, goto notification log."})
+
+                    log_lines = str(e).splitlines()
+                    notification_debug_log += log_lines
+
+                    with app.app_context():
+                        app.config['watch_check_update_SIGNAL'].send(app_context=app, watch_uuid=n_object.get('uuid'))
+
+                # Process notifications
+                notification_debug_log+= ["{} - SENDING - {}".format(now.strftime("%Y/%m/%d %H:%M:%S,000"), json.dumps(sent_obj))]
+                # Trim the log length
+                notification_debug_log = notification_debug_log[-100:]
 
-                log_lines = str(e).splitlines()
-                notification_debug_log += log_lines
 
-            # Process notifications
-            notification_debug_log+= ["{} - SENDING - {}".format(now.strftime("%Y/%m/%d %H:%M:%S,000"), json.dumps(sent_obj))]
-            # Trim the log length
-            notification_debug_log = notification_debug_log[-100:]
 
 # Threaded runner, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():

changedetectionio/forms.py

@@ -224,27 +224,37 @@ class StringDictKeyValue(StringField):
 
     def _value(self):
         if self.data:
-            output = u''
-            for k in self.data.keys():
-                output += "{}: {}\r\n".format(k, self.data[k])
-
+            output = ''
+            for k, v in self.data.items():
+                output += f"{k}: {v}\r\n"
             return output
         else:
-            return u''
+            return ''
 
-    # incoming
+    # incoming data processing + validation
     def process_formdata(self, valuelist):
+        self.data = {}
+        errors = []
         if valuelist:
-            self.data = {}
-            # Remove empty strings
-            cleaned = list(filter(None, valuelist[0].split("\n")))
-            for s in cleaned:
-                parts = s.strip().split(':', 1)
-                if len(parts) == 2:
-                    self.data.update({parts[0].strip(): parts[1].strip()})
+            # Remove empty strings (blank lines)
+            cleaned = [line.strip() for line in valuelist[0].split("\n") if line.strip()]
+            for idx, s in enumerate(cleaned, start=1):
+                if ':' not in s:
+                    errors.append(f"Line {idx} is missing a ':' separator.")
+                    continue
+                parts = s.split(':', 1)
+                key = parts[0].strip()
+                value = parts[1].strip()
 
-        else:
-            self.data = {}
+                if not key:
+                    errors.append(f"Line {idx} has an empty key.")
+                if not value:
+                    errors.append(f"Line {idx} has an empty value.")
+
+                self.data[key] = value
+
+        if errors:
+            raise ValidationError("Invalid input:\n" + "\n".join(errors))
 
 class ValidateContentFetcherIsReady(object):
     """

changedetectionio/html_tools.py

@@ -309,10 +309,10 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
         soup = BeautifulSoup(content, 'html.parser')
 
         if ensure_is_ldjson_info_type:
-            bs_result = soup.findAll('script', {"type": "application/ld+json"})
+            bs_result = soup.find_all('script', {"type": "application/ld+json"})
         else:
-            bs_result = soup.findAll('script')
-        bs_result += soup.findAll('body')
+            bs_result = soup.find_all('script')
+        bs_result += soup.find_all('body')
 
         bs_jsons = []
         for result in bs_result:
@@ -436,55 +436,27 @@ def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False
     return re.sub(pattern, repl, html_content)
 
 
-def html_to_text_sub_worker(conn, html_content: str, render_anchor_tag_content=False, is_rss=False):
+# NOTE!! ANYTHING LIBXML, HTML5LIB ETC WILL CAUSE SOME SMALL MEMORY LEAK IN THE LOCAL "LIB" IMPLEMENTATION OUTSIDE PYTHON
 
+
+def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=False, timeout=10) -> str:
     from inscriptis import get_text
     from inscriptis.model.config import ParserConfig
 
-    """Converts html string to a string with just the text. If ignoring
-    rendering anchor tag content is enable, anchor tag content are also
-    included in the text
-
-    :param html_content: string with html content
-    :param render_anchor_tag_content: boolean flag indicating whether to extract
-    hyperlinks (the anchor tag content) together with text. This refers to the
-    'href' inside 'a' tags.
-    Anchor tag content is rendered in the following manner:
-    '[ text ](anchor tag content)'
-    :return: extracted text from the HTML
-    """
-    #  if anchor tag content flag is set to True define a config for
-    #  extracting this content
     if render_anchor_tag_content:
         parser_config = ParserConfig(
             annotation_rules={"a": ["hyperlink"]},
             display_links=True
         )
-    # otherwise set config to None/default
     else:
         parser_config = None
 
-    # RSS Mode - Inscriptis will treat `title` as something else.
-    # Make it as a regular block display element (//item/title)
-    # This is a bit of a hack - the real way it to use XSLT to convert it to HTML #1874
     if is_rss:
         html_content = re.sub(r'<title([\s>])', r'<h1\1', html_content)
         html_content = re.sub(r'</title>', r'</h1>', html_content)
 
     text_content = get_text(html_content, config=parser_config)
-    conn.send(text_content)
-    conn.close()
-
-# NOTE!! ANYTHING LIBXML, HTML5LIB ETC WILL CAUSE SOME SMALL MEMORY LEAK IN THE LOCAL "LIB" IMPLEMENTATION OUTSIDE PYTHON
-def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=False):
-    from multiprocessing import Process, Pipe
-
-    parent_conn, child_conn = Pipe()
-    p = Process(target=html_to_text_sub_worker, args=(child_conn, html_content, render_anchor_tag_content, is_rss))
-    p.start()
-    text = parent_conn.recv()
-    p.join()
-    return text
+    return text_content
 
 # Does LD+JSON exist with a @type=='product' and a .price set anywhere?
 def has_ldjson_product_info(content):

changedetectionio/model/Watch.py

@@ -1,3 +1,5 @@
+from blinker import signal
+
 from changedetectionio.strtobool import strtobool
 from changedetectionio.safe_jinja import render as jinja_render
 from . import watch_base
@@ -41,6 +43,7 @@ class model(watch_base):
         self.__datastore_path = kw.get('datastore_path')
         if kw.get('datastore_path'):
             del kw['datastore_path']
+            
         super(model, self).__init__(*arg, **kw)
         if kw.get('default'):
             self.update(kw['default'])
@@ -60,6 +63,10 @@ class model(watch_base):
 
         return False
 
+    @property
+    def has_unviewed(self):
+        return int(self.newest_history_key) > int(self['last_viewed']) and self.__history_n >= 2
+
     def ensure_data_dir_exists(self):
         if not os.path.isdir(self.watch_data_dir):
             logger.debug(f"> Creating data dir {self.watch_data_dir}")
@@ -120,6 +127,10 @@ class model(watch_base):
             'remote_server_reply': None,
             'track_ldjson_price_data': None
         })
+        watch_check_update = signal('watch_check_update')
+        if watch_check_update:
+            watch_check_update.send(watch_uuid=self.get('uuid'))
+
         return
 
     @property
@@ -648,3 +659,45 @@ class model(watch_base):
             if step_n:
                 available.append(step_n.group(1))
         return available
+
+    def compile_error_texts(self, has_proxies=None):
+        """Compile error texts for this watch.
+        Accepts has_proxies parameter to ensure it works even outside app context"""
+        from flask import (
+            Markup, url_for
+        )
+
+        output = []  # Initialize as list since we're using append
+        last_error = self.get('last_error','')
+
+        try:
+            url_for('settings.settings_page')
+        except Exception as e:
+            has_app_context = False
+        else:
+            has_app_context = True
+
+        # has app+request context, we can use url_for()
+        if has_app_context:
+            if last_error:
+                if '403' in last_error:
+                    if has_proxies:
+                        output.append(str(Markup(f"{last_error} - <a href=\"{url_for('settings.settings_page', uuid=self.get('uuid'))}\">Try other proxies/location</a>&nbsp;'")))
+                    else:
+                        output.append(str(Markup(f"{last_error} - <a href=\"{url_for('settings.settings_page', uuid=self.get('uuid'))}\">Try adding external proxies/locations</a>&nbsp;'")))
+                else:
+                    output.append(str(Markup(last_error)))
+
+            if self.get('last_notification_error'):
+                output.append(str(Markup(f"<div class=\"notification-error\"><a href=\"{url_for('settings.notification_logs')}\">{ self.get('last_notification_error') }</a></div>")))
+
+        else:
+            # Lo_Fi version
+            if last_error:
+                output.append(str(Markup(last_error)))
+            if self.get('last_notification_error'):
+                output.append(str(Markup(self.get('last_notification_error'))))
+
+        res = "\n".join(output)
+        return res
+

changedetectionio/model/__init__.py

@@ -36,6 +36,7 @@ class watch_base(dict):
             'include_filters': [],
             'last_checked': 0,
             'last_error': False,
+            'last_notification_error': None,
             'last_viewed': 0,  # history key value of the last viewed via the [diff] link
             'method': 'GET',
             'notification_alert_count': 0,

changedetectionio/notification/handler.py

@@ -2,10 +2,8 @@
 import time
 import apprise
 from loguru import logger
-
 from .apprise_plugin.assets import apprise_asset, APPRISE_AVATAR_URL
 
-
 def process_notification(n_object, datastore):
     from changedetectionio.safe_jinja import render as jinja_render
     from . import default_notification_format_for_watch, default_notification_format, valid_notification_formats

changedetectionio/pluggy_interface.py

@@ -0,0 +1,82 @@
+import pluggy
+import os
+import importlib
+import sys
+
+# Global plugin namespace for changedetection.io
+PLUGIN_NAMESPACE = "changedetectionio"
+
+hookspec = pluggy.HookspecMarker(PLUGIN_NAMESPACE)
+hookimpl = pluggy.HookimplMarker(PLUGIN_NAMESPACE)
+
+
+class ChangeDetectionSpec:
+    """Hook specifications for extending changedetection.io functionality."""
+
+    @hookspec
+    def ui_edit_stats_extras(watch):
+        """Return HTML content to add to the stats tab in the edit view.
+        
+        Args:
+            watch: The watch object being edited
+            
+        Returns:
+            str: HTML content to be inserted in the stats tab
+        """
+        pass
+
+
+# Set up Plugin Manager
+plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE)
+
+# Register hookspecs
+plugin_manager.add_hookspecs(ChangeDetectionSpec)
+
+# Load plugins from subdirectories
+def load_plugins_from_directories():
+    # Dictionary of directories to scan for plugins
+    plugin_dirs = {
+        'conditions': os.path.join(os.path.dirname(__file__), 'conditions', 'plugins'),
+        # Add more plugin directories here as needed
+    }
+    
+    # Note: Removed the direct import of example_word_count_plugin as it's now in the conditions/plugins directory
+    
+    for dir_name, dir_path in plugin_dirs.items():
+        if not os.path.exists(dir_path):
+            continue
+            
+        # Get all Python files (excluding __init__.py)
+        for filename in os.listdir(dir_path):
+            if filename.endswith(".py") and filename != "__init__.py":
+                module_name = filename[:-3]  # Remove .py extension
+                module_path = f"changedetectionio.{dir_name}.plugins.{module_name}"
+                
+                try:
+                    module = importlib.import_module(module_path)
+                    # Register the plugin with pluggy
+                    plugin_manager.register(module, module_name)
+                except (ImportError, AttributeError) as e:
+                    print(f"Error loading plugin {module_name}: {e}")
+
+# Load plugins
+load_plugins_from_directories()
+
+# Discover installed plugins from external packages (if any)
+plugin_manager.load_setuptools_entrypoints(PLUGIN_NAMESPACE)
+
+# Helper function to collect UI stats extras from all plugins
+def collect_ui_edit_stats_extras(watch):
+    """Collect and combine HTML content from all plugins that implement ui_edit_stats_extras"""
+    extras_content = []
+    
+    # Get all plugins that implement the ui_edit_stats_extras hook
+    results = plugin_manager.hook.ui_edit_stats_extras(watch=watch)
+    
+    # If we have results, add them to our content
+    if results:
+        for result in results:
+            if result:  # Skip empty results
+                extras_content.append(result)
+            
+    return "\n".join(extras_content) if extras_content else ""

changedetectionio/processors/__init__.py

@@ -89,7 +89,7 @@ class difference_detection_processor():
                 proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')
                 logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}' for {url}")
             else:
-                logger.debug(f"Skipping adding proxy data when custom Browser endpoint is specified. ")
+                logger.debug("Skipping adding proxy data when custom Browser endpoint is specified. ")
 
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)

changedetectionio/processors/restock_diff/processor.py

@@ -79,7 +79,7 @@ def get_itemprop_availability(html_content) -> Restock:
     # First phase, dead simple scanning of anything that looks useful
     value = Restock()
     if data:
-        logger.debug(f"Using jsonpath to find price/availability/etc")
+        logger.debug("Using jsonpath to find price/availability/etc")
         price_parse = parse('$..(price|Price)')
         pricecurrency_parse = parse('$..(pricecurrency|currency|priceCurrency )')
         availability_parse = parse('$..(availability|Availability)')
@@ -110,7 +110,7 @@ def get_itemprop_availability(html_content) -> Restock:
 
         # Second, go dig OpenGraph which is something that jsonpath_ng cant do because of the tuples and double-dots (:)
         if not value.get('price') or value.get('availability'):
-            logger.debug(f"Alternatively digging through OpenGraph properties for restock/price info..")
+            logger.debug("Alternatively digging through OpenGraph properties for restock/price info..")
             jsonpath_expr = parse('$..properties')
 
             for match in jsonpath_expr.find(data):

changedetectionio/processors/text_json_diff/__init__.py

@@ -15,7 +15,7 @@ def _task(watch, update_handler):
     except FilterNotFoundInResponse as e:
         text_after_filter = f"Filter not found in HTML: {str(e)}"
     except ReplyWithContentButNoText as e:
-        text_after_filter = f"Filter found but no text (empty result)"
+        text_after_filter = "Filter found but no text (empty result)"
     except Exception as e:
         text_after_filter = f"Error: {str(e)}"
 

changedetectionio/realtime/__init__.py

@@ -0,0 +1,3 @@
+"""
+Socket.IO realtime updates module for changedetection.io
+"""

changedetectionio/realtime/socket_server.py

@@ -0,0 +1,283 @@
+import timeago
+from flask_socketio import SocketIO
+
+import time
+import os
+from loguru import logger
+from blinker import signal
+
+from changedetectionio import strtobool
+
+class SignalHandler:
+    """A standalone class to receive signals"""
+    def __init__(self, socketio_instance, datastore):
+        self.socketio_instance = socketio_instance
+        self.datastore = datastore
+
+        # Connect to the watch_check_update signal
+        from changedetectionio.flask_app import watch_check_update as wcc
+        wcc.connect(self.handle_signal, weak=False)
+        logger.info("SignalHandler: Connected to signal from direct import")
+        
+        # Connect to the queue_length signal
+        queue_length_signal = signal('queue_length')
+        queue_length_signal.connect(self.handle_queue_length, weak=False)
+        logger.info("SignalHandler: Connected to queue_length signal")
+
+
+        # Create and start the queue update thread using gevent
+        import gevent
+        logger.info("Using gevent for polling thread")
+        self.polling_emitter_thread = gevent.spawn(self.polling_emit_running_or_queued_watches)
+        
+        # Store the thread reference in socketio for clean shutdown
+        self.socketio_instance.polling_emitter_thread = self.polling_emitter_thread
+
+    def handle_signal(self, *args, **kwargs):
+        logger.trace(f"SignalHandler: Signal received with {len(args)} args and {len(kwargs)} kwargs")
+        # Safely extract the watch UUID from kwargs
+        watch_uuid = kwargs.get('watch_uuid')
+        app_context = kwargs.get('app_context')
+
+        if watch_uuid:
+            # Get the watch object from the datastore
+            watch = self.datastore.data['watching'].get(watch_uuid)
+            if watch:
+                if app_context:
+                    #note
+                    with app_context.app_context():
+                        with app_context.test_request_context():
+                            # Forward to handle_watch_update with the watch parameter
+                            handle_watch_update(self.socketio_instance, watch=watch, datastore=self.datastore)
+                else:
+                    handle_watch_update(self.socketio_instance, watch=watch, datastore=self.datastore)
+
+                logger.info(f"Signal handler processed watch UUID {watch_uuid}")
+            else:
+                logger.warning(f"Watch UUID {watch_uuid} not found in datastore")
+
+    def handle_queue_length(self, *args, **kwargs):
+        """Handle queue_length signal and emit to all clients"""
+        try:
+            queue_length = kwargs.get('length', 0)
+            logger.debug(f"SignalHandler: Queue length update received: {queue_length}")
+            
+            # Emit the queue size to all connected clients
+            self.socketio_instance.emit("queue_size", {
+                "q_length": queue_length,
+                "event_timestamp": time.time()
+            })
+            
+        except Exception as e:
+            logger.error(f"Socket.IO error in handle_queue_length: {str(e)}")
+
+
+    def polling_emit_running_or_queued_watches(self):
+        """Greenlet that periodically updates the browser/frontend with current state of who is being checked or queued
+        This is because sometimes the browser page could reload (like on clicking on a link) but the data is old
+        """
+        logger.info("Queue update greenlet started")
+
+        # Import the watch_check_update signal, update_q, and running_update_threads here to avoid circular imports
+        from changedetectionio.flask_app import app, running_update_threads
+        watch_check_update = signal('watch_check_update')
+        
+        # Use gevent sleep for non-blocking operation
+        from gevent import sleep as gevent_sleep
+
+        # Get the stop event from the socketio instance
+        stop_event = self.socketio_instance.stop_event if hasattr(self.socketio_instance, 'stop_event') else None
+
+        # Run until explicitly stopped
+        while stop_event is None or not stop_event.is_set():
+            try:
+                # For each item in the queue, send a signal, so we update the UI
+                for t in running_update_threads:
+                    if hasattr(t, 'current_uuid') and t.current_uuid:
+                        logger.debug(f"Sending update for {t.current_uuid}")
+                        # Send with app_context to ensure proper URL generation
+                        with app.app_context():
+                            watch_check_update.send(app_context=app, watch_uuid=t.current_uuid)
+                        # Yield control back to gevent after each send to prevent blocking
+                        gevent_sleep(0.1)  # Small sleep to yield control
+                    
+                    # Check if we need to stop in the middle of processing
+                    if stop_event is not None and stop_event.is_set():
+                        break
+
+                # Sleep between polling/update cycles
+                gevent_sleep(2)
+
+            except Exception as e:
+                logger.error(f"Error in queue update greenlet: {str(e)}")
+                # Sleep a bit to avoid flooding logs in case of persistent error
+                gevent_sleep(0.5)
+
+        logger.info("Queue update greenlet stopped")
+
+
+def handle_watch_update(socketio, **kwargs):
+    """Handle watch update signal from blinker"""
+    try:
+        watch = kwargs.get('watch')
+        datastore = kwargs.get('datastore')
+
+        # Emit the watch update to all connected clients
+        from changedetectionio.flask_app import running_update_threads, update_q
+        from changedetectionio.flask_app import _jinja2_filter_datetime
+
+        # Get list of watches that are currently running
+        running_uuids = []
+        for t in running_update_threads:
+            if hasattr(t, 'current_uuid') and t.current_uuid:
+                running_uuids.append(t.current_uuid)
+
+        # Get list of watches in the queue
+        queue_list = []
+        for q_item in update_q.queue:
+            if hasattr(q_item, 'item') and 'uuid' in q_item.item:
+                queue_list.append(q_item.item['uuid'])
+
+        error_texts = ""
+        # Get the error texts from the watch
+        error_texts = watch.compile_error_texts()
+
+        # Create a simplified watch data object to send to clients
+        watch_data = {
+            'checking_now': True if watch.get('uuid') in running_uuids else False,
+            'fetch_time': watch.get('fetch_time'),
+            'has_error': True if error_texts else False,
+            'last_changed': watch.get('last_changed'),
+            'last_checked': watch.get('last_checked'),
+            'error_text': error_texts,
+            'last_checked_text': _jinja2_filter_datetime(watch),
+            'last_changed_text': timeago.format(int(watch['last_changed']), time.time()) if watch.history_n >= 2 and int(watch.get('last_changed', 0)) > 0 else 'Not yet',
+            'queued': True if watch.get('uuid') in queue_list else False,
+            'paused': True if watch.get('paused') else False,
+            'notification_muted': True if watch.get('notification_muted') else False,
+            'unviewed': watch.has_unviewed,
+            'uuid': watch.get('uuid'),
+            'event_timestamp': time.time()
+        }
+
+        errored_count =0
+        for uuid, watch in datastore.data['watching'].items():
+            if watch.get('last_error'):
+                errored_count += 1
+
+        general_stats = {
+            'count_errors': errored_count,
+            'has_unviewed': datastore.has_unviewed
+        }
+
+        # Debug what's being emitted
+        #logger.debug(f"Emitting 'watch_update' event for {watch.get('uuid')}, data: {watch_data}")
+        
+        # Emit to all clients (no 'broadcast' parameter needed - it's the default behavior)
+        socketio.emit("watch_update", {'watch': watch_data, 'general_stats': general_stats})
+        
+        # Log after successful emit
+        #logger.info(f"Socket.IO: Emitted update for watch {watch.get('uuid')}, Checking now: {watch_data['checking_now']}")
+
+    except Exception as e:
+        logger.error(f"Socket.IO error in handle_watch_update: {str(e)}")
+
+
+def init_socketio(app, datastore):
+    """Initialize SocketIO with the main Flask app"""
+    # Use the threading async_mode instead of eventlet
+    # This avoids the need for monkey patching eventlet,
+    # Which leads to problems with async playwright etc
+    async_mode = 'gevent'
+    logger.info(f"Using {async_mode} mode for Socket.IO")
+
+    # Restrict SocketIO CORS to same origin by default, can be overridden with env var
+    cors_origins = os.environ.get('SOCKETIO_CORS_ORIGINS', None)
+    
+    socketio = SocketIO(app,
+                      async_mode=async_mode,
+                      cors_allowed_origins=cors_origins,  # None means same-origin only
+                      logger=strtobool(os.getenv('SOCKETIO_LOGGING', 'False')),
+                      engineio_logger=strtobool(os.getenv('SOCKETIO_LOGGING', 'False')))
+
+    # Set up event handlers
+    @socketio.on('connect')
+    def handle_connect():
+        """Handle client connection"""
+        from changedetectionio.auth_decorator import login_optionally_required
+        from flask import request
+        from flask_login import current_user
+        from changedetectionio.flask_app import update_q
+
+        # Access datastore from socketio
+        datastore = socketio.datastore
+
+        # Check if authentication is required and user is not authenticated
+        has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False)
+        if has_password_enabled and not current_user.is_authenticated:
+            logger.warning("Socket.IO: Rejecting unauthenticated connection")
+            return False  # Reject the connection
+
+        # Send the current queue size to the newly connected client
+        try:
+            queue_size = update_q.qsize()
+            socketio.emit("queue_size", {
+                "q_length": queue_size,
+                "event_timestamp": time.time()
+            }, room=request.sid)  # Send only to this client
+            logger.debug(f"Socket.IO: Sent initial queue size {queue_size} to new client")
+        except Exception as e:
+            logger.error(f"Socket.IO error sending initial queue size: {str(e)}")
+
+        logger.info("Socket.IO: Client connected")
+
+    @socketio.on('disconnect')
+    def handle_disconnect():
+        """Handle client disconnection"""
+        logger.info("Socket.IO: Client disconnected")
+
+    # Create a dedicated signal handler that will receive signals and emit them to clients
+    signal_handler = SignalHandler(socketio, datastore)
+
+    # Store the datastore reference on the socketio object for later use
+    socketio.datastore = datastore
+    
+    # Create a stop event for our queue update thread using gevent Event
+    import gevent.event
+    stop_event = gevent.event.Event()
+    socketio.stop_event = stop_event
+
+    
+    # Add a shutdown method to the socketio object
+    def shutdown():
+        """Shutdown the SocketIO server gracefully"""
+        try:
+            logger.info("Socket.IO: Shutting down server...")
+            
+            # Signal the queue update thread to stop
+            if hasattr(socketio, 'stop_event'):
+                socketio.stop_event.set()
+                logger.info("Socket.IO: Signaled queue update thread to stop")
+            
+            # Wait for the greenlet to exit (with timeout)
+            if hasattr(socketio, 'polling_emitter_thread'):
+                try:
+                    # For gevent greenlets
+                    socketio.polling_emitter_thread.join(timeout=5)
+                    logger.info("Socket.IO: Queue update greenlet joined successfully")
+                except Exception as e:
+                    logger.error(f"Error joining greenlet: {str(e)}")
+                    logger.info("Socket.IO: Queue update greenlet did not exit in time")
+            
+            # Close any remaining client connections
+            #if hasattr(socketio, 'server'):
+            #    socketio.server.disconnect()
+            logger.info("Socket.IO: Server shutdown complete")
+        except Exception as e:
+            logger.error(f"Socket.IO error during shutdown: {str(e)}")
+    
+    # Attach the shutdown method to the socketio object
+    socketio.shutdown = shutdown
+
+    logger.info("Socket.IO initialized and attached to main Flask app")
+    return socketio

changedetectionio/run_basic_tests.sh

@@ -38,6 +38,9 @@ pytest tests/test_backend.py
 pytest tests/test_rss.py
 pytest tests/test_unique_lines.py
 
+# Try high concurrency
+FETCH_WORKERS=130 pytest  tests/test_history_consistency.py -v -l
+
 # Check file:// will pickup a file when enabled
 echo "Hello world" > /tmp/test-file.txt
 ALLOW_FILE_URI=yes pytest tests/test_security.py

changedetectionio/run_proxy_tests.sh

@@ -82,3 +82,25 @@ done
 
 
 docker kill squid-one squid-two squid-custom
+
+# Test that the UI is returning the correct error message when a proxy is not available
+
+# Requests
+docker run --network changedet-network \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_proxy_noconnect.py'
+
+# Playwright
+docker run --network changedet-network \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py'
+
+# Puppeteer fast
+docker run --network changedet-network \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && FAST_PUPPETEER_CHROME_FETCHER=1 PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py'
+
+# Selenium
+docker run --network changedet-network \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && WEBDRIVER_URL=http://selenium:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py'

changedetectionio/static/js/browser-steps.js

@@ -211,7 +211,14 @@ $(document).ready(function () {
                     $('input[type=text]', first_available).first().val(x['xpath']);
                     $('input[placeholder="Value"]', first_available).addClass('ok').click().focus();
                     found_something = true;
-                } else {
+                }
+                else if (x['tagName'] === 'select') {
+                    $('select', first_available).val('<select> by option text').change();
+                    $('input[type=text]', first_available).first().val(x['xpath']);
+                    $('input[placeholder="Value"]', first_available).addClass('ok').click().focus();
+                    found_something = true;
+                }
+                else {
                     // There's no good way (that I know) to find if this
                     // see https://stackoverflow.com/questions/446892/how-to-find-event-listeners-on-a-dom-node-in-javascript-or-in-debugging
                     // https://codepen.io/azaslavsky/pen/DEJVWv
@@ -251,6 +258,10 @@ $(document).ready(function () {
                 400: function () {
                     // More than likely the CSRF token was lost when the server restarted
                     alert("There was a problem processing the request, please reload the page.");
+                },
+                401: function (err) {
+                    // This will be a custom error
+                    alert(err.responseText);
                 }
             }
         }).done(function (data) {

changedetectionio/static/js/realtime.js

@@ -0,0 +1,115 @@
+// Socket.IO client-side integration for changedetection.io
+
+$(document).ready(function () {
+
+    function bindAjaxHandlerButtonsEvents() {
+        $('.ajax-op').on('click.ajaxHandlerNamespace', function (e) {
+            e.preventDefault();
+            $.ajax({
+                type: "POST",
+                url: ajax_toggle_url,
+                data: {'op': $(this).data('op'), 'uuid': $(this).closest('tr').data('watch-uuid')},
+                statusCode: {
+                    400: function () {
+                        // More than likely the CSRF token was lost when the server restarted
+                        alert("There was a problem processing the request, please reload the page.");
+                    }
+                }
+            });
+            return false;
+        });
+    }
+
+
+    // Only try to connect if authentication isn't required or user is authenticated
+    // The 'is_authenticated' variable will be set in the template
+    if (typeof is_authenticated !== 'undefined' ? is_authenticated : true) {
+        // Try to create the socket connection to the SocketIO server - if it fails, the site will still work normally
+        try {
+            // Connect to Socket.IO on the same host/port, with path from template
+            const socket = io({
+                path: socketio_url,  // This will be the path prefix like "/app/socket.io" from the template
+                transports: ['polling', 'websocket'],  // Try WebSocket but fall back to polling
+                reconnectionDelay: 1000,
+                reconnectionAttempts: 15
+            });
+
+            // Connection status logging
+            socket.on('connect', function () {
+                console.log('Socket.IO connected with path:', socketio_url);
+                console.log('Socket transport:', socket.io.engine.transport.name);
+                bindAjaxHandlerButtonsEvents();
+            });
+
+            socket.on('connect_error', function(error) {
+                console.error('Socket.IO connection error:', error);
+            });
+
+            socket.on('connect_timeout', function() {
+                console.error('Socket.IO connection timeout');
+            });
+
+            socket.on('error', function(error) {
+                console.error('Socket.IO error:', error);
+            });
+
+            socket.on('disconnect', function (reason) {
+                console.log('Socket.IO disconnected, reason:', reason);
+                $('.ajax-op').off('.ajaxHandlerNamespace')
+            });
+
+            socket.on('queue_size', function (data) {
+                console.log(`${data.event_timestamp} - Queue size update: ${data.q_length}`);
+                // Update queue size display if implemented in the UI
+            })
+
+            // Listen for periodically emitted watch data
+            console.log('Adding watch_update event listener');
+
+            socket.on('watch_update', function (data) {
+                const watch = data.watch;
+                const general_stats = data.general_stats;
+
+                // Log the entire watch object for debugging
+                console.log('!!! WATCH UPDATE EVENT RECEIVED !!!');
+                console.log(`${watch.event_timestamp} - Watch update ${watch.uuid} - Checking now - ${watch.checking_now} - UUID in URL ${window.location.href.includes(watch.uuid)}`);
+                console.log('Watch data:', watch);
+                console.log('General stats:', general_stats);
+                
+                // Updating watch table rows
+                const $watchRow = $('tr[data-watch-uuid="' + watch.uuid + '"]');
+                console.log('Found watch row elements:', $watchRow.length);
+                
+                if ($watchRow.length) {
+                    $($watchRow).toggleClass('checking-now', watch.checking_now);
+                    $($watchRow).toggleClass('queued', watch.queued);
+                    $($watchRow).toggleClass('unviewed', watch.unviewed);
+                    $($watchRow).toggleClass('has-error', watch.has_error);
+                    $($watchRow).toggleClass('notification_muted', watch.notification_muted);
+                    $($watchRow).toggleClass('paused', watch.paused);
+
+                    $('td.title-col .error-text', $watchRow).html(watch.error_text)
+
+                    $('td.last-changed', $watchRow).text(watch.last_checked_text)
+
+                    $('td.last-checked .innertext', $watchRow).text(watch.last_checked_text)
+                    $('td.last-checked', $watchRow).data('timestamp', watch.last_checked).data('fetchduration', watch.fetch_time);
+                    $('td.last-checked', $watchRow).data('eta_complete', watch.last_checked + watch.fetch_time);
+                    
+                    console.log('Updated UI for watch:', watch.uuid);
+                }
+
+                // Tabs at bottom of list
+                $('#post-list-mark-views').toggleClass("has-unviewed", general_stats.has_unviewed);
+                $('#post-list-with-errors').toggleClass("has-error", general_stats.count_errors !== 0)
+                $('#post-list-with-errors a').text(`xxxxWith errors (${ general_stats.count_errors })`);
+
+                $('body').toggleClass('checking-now', watch.checking_now && window.location.href.includes(watch.uuid));
+            });
+
+        } catch (e) {
+            // If Socket.IO fails to initialize, just log it and continue
+            console.log('Socket.IO initialization error:', e);
+        }
+    }
+});

changedetectionio/static/js/watch-overview.js

@@ -68,7 +68,7 @@ $(function () {
             if (eta_complete + 2 > nowtimeserver && fetch_duration > 3) {
                 const remaining_seconds = Math.abs(eta_complete) - nowtimeserver - 1;
 
-                let r = (1.0 - (remaining_seconds / fetch_duration)) * 100;
+                let r = Math.round((1.0 - (remaining_seconds / fetch_duration)) * 100);
                 if (r < 10) {
                     r = 10;
                 }
@@ -76,8 +76,8 @@ $(function () {
                     r = 100;
                 }
                 $(this).css('background-size', `${r}% 100%`);
-                //$(this).text(`${r}% remain ${remaining_seconds}`);
             } else {
+                // Snap to full complete
                 $(this).css('background-size', `100% 100%`);
             }
         });

changedetectionio/static/styles/scss/parts/_socket.scss

@@ -0,0 +1,31 @@
+// Styles for Socket.IO real-time updates
+body.checking-now {
+  #checking-now-fixed-tab {
+    display: block !important;
+  }
+}
+
+#checking-now-fixed-tab {
+  background: #ccc;
+  border-radius: 5px;
+  bottom: 0;
+  color: var(--color-text);
+  display: none;
+  font-size: 0.8rem;
+  left: 0;
+  padding: 5px;
+  position: fixed;
+}
+
+#post-list-buttons {
+  #post-list-with-errors.has-error {
+    display: inline-block !important;
+  }
+  #post-list-mark-views.has-unviewed {
+    display: inline-block !important;
+  }
+}
+
+
+
+

changedetectionio/static/styles/scss/parts/_watch_table.scss

@@ -0,0 +1,118 @@
+/* table related */
+.watch-table {
+  width: 100%;
+  font-size: 80%;
+
+  tr {
+    &.unviewed {
+      font-weight: bold;
+    }
+    color: var(--color-watch-table-row-text);
+  }
+
+
+  td {
+    white-space: nowrap;
+
+    &.title-col {
+      word-break: break-all;
+      white-space: normal;
+    }
+  }
+
+
+  th {
+    white-space: nowrap;
+
+    a {
+      font-weight: normal;
+
+      &.active {
+        font-weight: bolder;
+      }
+
+      &.inactive {
+        .arrow {
+          display: none;
+        }
+      }
+    }
+  }
+
+  .title-col a[target="_blank"]::after,
+  .current-diff-url::after {
+    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
+    margin: 0 3px 0 5px;
+  }
+
+
+  /* Row with 'checking-now' */
+  tr.checking-now {
+    td:first-child {
+        position: relative;
+    }
+
+    td:first-child::before {
+        content: "";
+        position: absolute;
+        top: 0;
+        bottom: 0;
+        left: 0;
+        width: 3px;
+        background-color: #293eff;
+    }
+
+    td.last-checked {
+      .spinner-wrapper {
+        display: inline-block !important;
+      }
+
+      .innertext {
+        display: none !important;
+      }
+    }
+  }
+
+  tr.queued {
+    a.recheck {
+      display: none !important;
+    }
+
+    a.already-in-queue-button {
+      display: inline-block !important;
+    }
+  }
+
+  tr.paused {
+    a.pause-toggle {
+      &.state-on {
+        display: inline !important;
+      }
+
+      &.state-off {
+        display: none !important;
+      }
+    }
+  }
+
+  tr.notification_muted {
+    a.mute-toggle {
+      &.state-on {
+        display: inline !important;
+      }
+
+      &.state-off {
+        display: none !important;
+      }
+    }
+  }
+
+
+  tr.has-error {
+    color: var(--color-watch-table-error);
+    .error-text {
+      display: block !important;
+    }
+  }
+}
+

changedetectionio/static/styles/scss/styles.scss

@@ -13,8 +13,10 @@
 @import "parts/_menu";
 @import "parts/_love";
 @import "parts/preview_text_filter";
+@import "parts/_watch_table";
 @import "parts/_edit";
 @import "parts/_conditions_table";
+@import "parts/_socket";
 
 body {
   color: var(--color-text);
@@ -169,56 +171,6 @@ code {
   color: var(--color-text);
 }
 
-/* table related */
-.watch-table {
-  width: 100%;
-  font-size: 80%;
-
-  tr {
-    &.unviewed {
-      font-weight: bold;
-    }
-    &.error {
-      color: var(--color-watch-table-error);
-    }
-    color: var(--color-watch-table-row-text);
-  }
-
-
-  td {
-    white-space: nowrap;
-    &.title-col {
-      word-break: break-all;
-      white-space: normal;
-    }
-  }
-
-
-  th {
-    white-space: nowrap;
-
-    a {
-      font-weight: normal;
-
-      &.active {
-        font-weight: bolder;
-      }
-
-      &.inactive {
-        .arrow {
-          display: none;
-        }
-      }
-    }
-  }
-
-  .title-col a[target="_blank"]::after,
-  .current-diff-url::after {
-    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
-    margin: 0 3px 0 5px;
-  }
-}
-
 .inline-tag {
   white-space: nowrap;
   border-radius: 5px;

changedetectionio/static/styles/styles.css

@@ -523,6 +523,63 @@ body.preview-text-enabled {
   z-index: 3;
   box-shadow: 1px 1px 4px var(--color-shadow-jump); }
 
+/* table related */
+.watch-table {
+  width: 100%;
+  font-size: 80%;
+  /* Row with 'checking-now' */ }
+  .watch-table tr {
+    color: var(--color-watch-table-row-text); }
+    .watch-table tr.unviewed {
+      font-weight: bold; }
+  .watch-table td {
+    white-space: nowrap; }
+    .watch-table td.title-col {
+      word-break: break-all;
+      white-space: normal; }
+  .watch-table th {
+    white-space: nowrap; }
+    .watch-table th a {
+      font-weight: normal; }
+      .watch-table th a.active {
+        font-weight: bolder; }
+      .watch-table th a.inactive .arrow {
+        display: none; }
+  .watch-table .title-col a[target="_blank"]::after,
+  .watch-table .current-diff-url::after {
+    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
+    margin: 0 3px 0 5px; }
+  .watch-table tr.checking-now td:first-child {
+    position: relative; }
+  .watch-table tr.checking-now td:first-child::before {
+    content: "";
+    position: absolute;
+    top: 0;
+    bottom: 0;
+    left: 0;
+    width: 3px;
+    background-color: #293eff; }
+  .watch-table tr.checking-now td.last-checked .spinner-wrapper {
+    display: inline-block !important; }
+  .watch-table tr.checking-now td.last-checked .innertext {
+    display: none !important; }
+  .watch-table tr.queued a.recheck {
+    display: none !important; }
+  .watch-table tr.queued a.already-in-queue-button {
+    display: inline-block !important; }
+  .watch-table tr.paused a.pause-toggle.state-on {
+    display: inline !important; }
+  .watch-table tr.paused a.pause-toggle.state-off {
+    display: none !important; }
+  .watch-table tr.notification_muted a.mute-toggle.state-on {
+    display: inline !important; }
+  .watch-table tr.notification_muted a.mute-toggle.state-off {
+    display: none !important; }
+  .watch-table tr.has-error {
+    color: var(--color-watch-table-error); }
+    .watch-table tr.has-error .error-text {
+      display: block !important; }
+
 ul#conditions_match_logic {
   list-style: none; }
   ul#conditions_match_logic input, ul#conditions_match_logic label, ul#conditions_match_logic li {
@@ -623,6 +680,26 @@ ul#conditions_match_logic {
   .fieldlist_formfields .addRuleRow:hover, .fieldlist_formfields .removeRuleRow:hover, .fieldlist_formfields .verifyRuleRow:hover {
     background-color: #999; }
 
+body.checking-now #checking-now-fixed-tab {
+  display: block !important; }
+
+#checking-now-fixed-tab {
+  background: #ccc;
+  border-radius: 5px;
+  bottom: 0;
+  color: var(--color-text);
+  display: none;
+  font-size: 0.8rem;
+  left: 0;
+  padding: 5px;
+  position: fixed; }
+
+#post-list-buttons #post-list-with-errors.has-error {
+  display: inline-block !important; }
+
+#post-list-buttons #post-list-mark-views.has-unviewed {
+  display: inline-block !important; }
+
 body {
   color: var(--color-text);
   background: var(--color-background-page);
@@ -735,34 +812,6 @@ code {
   background: var(--color-background-code);
   color: var(--color-text); }
 
-/* table related */
-.watch-table {
-  width: 100%;
-  font-size: 80%; }
-  .watch-table tr {
-    color: var(--color-watch-table-row-text); }
-    .watch-table tr.unviewed {
-      font-weight: bold; }
-    .watch-table tr.error {
-      color: var(--color-watch-table-error); }
-  .watch-table td {
-    white-space: nowrap; }
-    .watch-table td.title-col {
-      word-break: break-all;
-      white-space: normal; }
-  .watch-table th {
-    white-space: nowrap; }
-    .watch-table th a {
-      font-weight: normal; }
-      .watch-table th a.active {
-        font-weight: bolder; }
-      .watch-table th a.inactive .arrow {
-        display: none; }
-  .watch-table .title-col a[target="_blank"]::after,
-  .watch-table .current-diff-url::after {
-    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
-    margin: 0 3px 0 5px; }
-
 .inline-tag, .watch-tag-list, .tracking-ldjson-price-data, .restock-label {
   white-space: nowrap;
   border-radius: 5px;

changedetectionio/store.py

@@ -17,6 +17,7 @@ import threading
 import time
 import uuid as uuid_builder
 from loguru import logger
+from blinker import signal
 
 from .processors import get_custom_watch_obj_for_processor
 from .processors.restock_diff import Restock
@@ -166,6 +167,10 @@ class ChangeDetectionStore:
         self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
         self.needs_write = True
 
+        watch_check_update = signal('watch_check_update')
+        if watch_check_update:
+            watch_check_update.send(watch_uuid=uuid)
+
     def remove_password(self):
         self.__data['settings']['application']['password'] = False
         self.needs_write = True

changedetectionio/templates/base.html

@@ -28,9 +28,14 @@
     <meta name="theme-color" content="#ffffff">
     <script>
         const csrftoken="{{ csrf_token() }}";
+        const socketio_url="{{ get_socketio_path() }}/socket.io";
+        const is_authenticated = {% if current_user.is_authenticated or not has_password %}true{% else %}false{% endif %};
     </script>
     <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
     <script src="{{url_for('static_content', group='js', filename='csrf.js')}}" defer></script>
+    <script src="{{url_for('static_content', group='js', filename='socket.io.min.js')}}" integrity="sha384-c79GN5VsunZvi+Q/WObgk2in0CbZsHnjEqvFxC5DxHn9lTfNce2WW6h2pH6u/kF+" crossorigin="anonymous"></script>
+    <script src="{{url_for('static_content', group='js', filename='realtime.js')}}" defer></script>
+    <script src="{{url_for('static_content', group='js', filename='timeago-init.js')}}" defer></script>
   </head>
 
   <body class="">
@@ -227,6 +232,8 @@
       {% block content %}{% endblock %}
     </section>
     <script src="{{url_for('static_content', group='js', filename='toggle-theme.js')}}" defer></script>
+
+    <div id="checking-now-fixed-tab" style="display: none;"><span class="spinner"></span><span>&nbsp;Checking now</span></div>
   </body>
 
 </html>

changedetectionio/templates/edit.html

@@ -239,7 +239,7 @@ Math: {{ 1 + 1 }}") }}
                                 </div>
                             </div>
                             <div id="browser-steps-fieldlist" >
-                                <span id="browser-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
+                                <span id="browser-seconds-remaining">Press "Play" to start.</span> <span style="font-size: 80%;"> (<a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
                                 {{ render_field(form.browser_steps) }}
                             </div>
                         </div>
@@ -383,13 +383,13 @@ Math: {{ 1 + 1 }}") }}
                     <div class="pure-control-group">
                         {% if watch_needs_selenium_or_playwright %}
                             {% if system_has_playwright_configured %}
-                                {%  if visual_selector_data_ready %}
                             <span class="pure-form-message-inline" id="visual-selector-heading">
                                 The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection. It automatically fills-in the filters in the "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab. Use <strong>Shift+Click</strong> to select multiple items.
                             </span>
 
                             <div id="selector-header">
                                 <a id="clear-selector" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Clear selection</a>
+                                <!-- visual selector IMG will try to load, it will either replace this or on error replace it with some handy text -->
                                 <i class="fetching-update-notice" style="font-size: 80%;">One moment, fetching screenshot and element information..</i>
                             </div>
                             <div id="selector-wrapper" style="display: none">
@@ -400,9 +400,6 @@ Math: {{ 1 + 1 }}") }}
                                 <canvas id="selector-canvas"></canvas>
                             </div>
                             <div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
-                            {% else %}
-                                <strong>Error, The Visual selector data is not ready, it needs to complete atleast one fetch, please queue the item and reload.</strong>
-                            {% endif %}
                         {% else %}
                             {# The watch needed chrome but system says that playwright is not ready #}
                             {{ playwright_warning() }}
@@ -453,6 +450,13 @@ Math: {{ 1 + 1 }}") }}
                         </tr>
                         </tbody>
                     </table>
+
+                    {% if ui_edit_stats_extras %}
+                    <div class="plugin-stats-extras"> <!-- from pluggy plugin -->
+                        {{ ui_edit_stats_extras|safe }}
+                    </div>
+                    {% endif %}
+
                     {% if watch.history_n %}
                         <p>
                              <a href="{{url_for('ui.ui_edit.watch_get_latest_html', uuid=uuid)}}" class="pure-button button-small">Download latest HTML snapshot</a>

changedetectionio/tests/custom_browser_url/test_custom_browser_url.py

@@ -7,7 +7,7 @@ from ..util import live_server_setup, wait_for_all_checks
 def do_test(client, live_server, make_test_use_extra_browser=False):
 
     # Grep for this string in the logs?
-    test_url = f"https://changedetection.io/ci-test.html?non-custom-default=true"
+    test_url = "https://changedetection.io/ci-test.html?non-custom-default=true"
     # "non-custom-default" should not appear in the custom browser connection
     custom_browser_name = 'custom browser URL'
 
@@ -51,7 +51,7 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
             url_for("ui.ui_edit.edit_page", uuid="first"),
             data={
                 # 'run_customer_browser_url_tests.sh' will search for this string to know if we hit the right browser container or not
-                  "url": f"https://changedetection.io/ci-test.html?custom-browser-search-string=1",
+                  "url": "https://changedetection.io/ci-test.html?custom-browser-search-string=1",
                   "tags": "",
                   "headers": "",
                   'fetch_backend': f"extra_browser_{custom_browser_name}",

changedetectionio/tests/proxy_list/test_proxy_noconnect.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+from flask import url_for
+from ..util import live_server_setup, wait_for_all_checks
+import os
+from ... import strtobool
+
+
+# Just to be sure the UI outputs the right error message on proxy connection failed
+# docker run -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
+# PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000 pytest tests/proxy_list/test_proxy_noconnect.py
+# FAST_PUPPETEER_CHROME_FETCHER=True PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000 pytest tests/proxy_list/test_proxy_noconnect.py
+# WEBDRIVER_URL=http://127.0.0.1:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py
+
+def test_proxy_noconnect_custom(client, live_server, measure_memory_usage):
+    live_server_setup(live_server)
+
+    # Goto settings, add our custom one
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={
+            "requests-time_between_check-minutes": 180,
+            "application-ignore_whitespace": "y",
+            "application-fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') or os.getenv("WEBDRIVER_URL") else 'html_requests',
+            "requests-extra_proxies-0-proxy_name": "custom-test-proxy",
+            # test:awesome is set in tests/proxy_list/squid-passwords.txt
+            "requests-extra_proxies-0-proxy_url": "http://127.0.0.1:3128",
+        },
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    test_url = "https://changedetection.io"
+    res = client.post(
+        url_for("ui.ui_views.form_quick_watch_add"),
+        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added in Paused state, saving will unpause" in res.data
+
+    options = {
+        "url": test_url,
+        "fetch_backend": "html_webdriver" if os.getenv('PLAYWRIGHT_DRIVER_URL') or os.getenv("WEBDRIVER_URL") else "html_requests",
+        "proxy": "ui-0custom-test-proxy",
+    }
+
+    res = client.post(
+        url_for("ui.ui_edit.edit_page", uuid="first", unpause_on_save=1),
+        data=options,
+        follow_redirects=True
+    )
+    assert b"unpaused" in res.data
+    import time
+    wait_for_all_checks(client)
+
+    # Requests default
+    check_string = b'Cannot connect to proxy'
+
+    if os.getenv('PLAYWRIGHT_DRIVER_URL') or strtobool(os.getenv('FAST_PUPPETEER_CHROME_FETCHER', 'False')) or os.getenv("WEBDRIVER_URL"):
+        check_string = b'ERR_PROXY_CONNECTION_FAILED'
+
+
+    res = client.get(url_for("watchlist.index"))
+    #with open("/tmp/debug.html", 'wb') as f:
+    #    f.write(res.data)
+    assert check_string in res.data

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -7,7 +7,7 @@ from changedetectionio.tests.util import live_server_setup, wait_for_all_checks,
 
 def set_response():
     import time
-    data = f"""<html>
+    data = """<html>
        <body>
      <h1>Awesome, you made it</h1>
      yeah the socks request worked

changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py

@@ -6,7 +6,7 @@ from changedetectionio.tests.util import live_server_setup, wait_for_all_checks
 
 def set_response():
     import time
-    data = f"""<html>
+    data = """<html>
        <body>
      <h1>Awesome, you made it</h1>
      yeah the socks request worked

changedetectionio/tests/realtime/test_socketio.py

@@ -0,0 +1,72 @@
+import asyncio
+import socketio
+from aiohttp import web
+
+SOCKETIO_URL = 'ws://localhost.localdomain:5005'
+SOCKETIO_PATH = "/socket.io"
+NUM_CLIENTS = 1
+
+clients = []
+shutdown_event = asyncio.Event()
+
+class WatchClient:
+    def __init__(self, client_id: int):
+        self.client_id = client_id
+        self.i_got_watch_update_event = False
+        self.sio = socketio.AsyncClient(reconnection_attempts=50, reconnection_delay=1)
+
+        @self.sio.event
+        async def connect():
+            print(f"[Client {self.client_id}] Connected")
+
+        @self.sio.event
+        async def disconnect():
+            print(f"[Client {self.client_id}] Disconnected")
+
+        @self.sio.on("watch_update")
+        async def on_watch_update(watch):
+            self.i_got_watch_update_event = True
+            print(f"[Client {self.client_id}] Received update: {watch}")
+
+    async def run(self):
+        try:
+            await self.sio.connect(SOCKETIO_URL, socketio_path=SOCKETIO_PATH, transports=["websocket", "polling"])
+            await self.sio.wait()
+        except Exception as e:
+            print(f"[Client {self.client_id}] Connection error: {e}")
+
+async def handle_check(request):
+    all_received = all(c.i_got_watch_update_event for c in clients)
+    result = "yes" if all_received else "no"
+    print(f"Received HTTP check — returning '{result}'")
+    shutdown_event.set()  # Signal shutdown
+    return web.Response(text=result)
+
+async def start_http_server():
+    app = web.Application()
+    app.add_routes([web.get('/did_all_clients_get_watch_update', handle_check)])
+    runner = web.AppRunner(app)
+    await runner.setup()
+    site = web.TCPSite(runner, '0.0.0.0', 6666)
+    await site.start()
+
+async def main():
+    #await start_http_server()
+
+    for i in range(NUM_CLIENTS):
+        client = WatchClient(i)
+        clients.append(client)
+        asyncio.create_task(client.run())
+
+    await shutdown_event.wait()
+
+    print("Shutting down...")
+    # Graceful disconnect
+    for c in clients:
+        await c.sio.disconnect()
+
+if __name__ == "__main__":
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("Interrupted")

changedetectionio/tests/restock/test_restock.py

@@ -14,6 +14,8 @@ from changedetectionio.notification import (
 def set_original_response():
     test_return_data = """<html>
        <body>
+       <section id=header style="padding: 50px; height: 350px">This is the header which should be ignored always - <span>add to cart</span></section>
+       <!-- stock-not-in-stock.js will ignore text in the first 300px, see elementIsInEyeBallRange(), sometimes "add to cart" and other junk is here -->
      Some initial text<br>
      <p>Which is across multiple lines</p>
      <br>
@@ -52,8 +54,6 @@ def test_restock_detection(client, live_server, measure_memory_usage):
 
     set_original_response()
     #assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
-
-    time.sleep(1)
     live_server_setup(live_server)
     #####################
     notification_url = url_for('test_notification_endpoint', _external=True).replace('http://localhost', 'http://changedet').replace('http', 'json')
@@ -84,7 +84,8 @@ def test_restock_detection(client, live_server, measure_memory_usage):
     # Is it correctly show as NOT in stock?
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'not-in-stock' in res.data
+    assert b'processor-restock_diff' in res.data # Should have saved in restock mode
+    assert b'not-in-stock' in res.data # should be out of stock
 
     # Is it correctly shown as in stock
     set_back_in_stock_response()

changedetectionio/tests/test_auth.py

@@ -4,14 +4,14 @@ import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 
-
+# test pages with http://username@password:foobar.com/ work
 def test_basic_auth(client, live_server, measure_memory_usage):
-
     live_server_setup(live_server)
 
-    # Add our URL to the import page
-    test_url = url_for('test_basicauth_method', _external=True).replace("//","//myuser:mypass@")
 
+    # This page will echo back any auth info
+    test_url = url_for('test_basicauth_method', _external=True).replace("//","//myuser:mypass@")
+    time.sleep(1)
     res = client.post(
         url_for("imports.import_page"),
         data={"urls": test_url},
@@ -34,4 +34,4 @@ def test_basic_auth(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
 
-    assert b'myuser mypass basic' in res.data
+    assert b'myuser mypass basic' in res.data

changedetectionio/tests/test_backend.py

@@ -114,7 +114,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
         # It should report nothing found (no new 'unviewed' class)
         res = client.get(url_for("watchlist.index"))
         assert b'unviewed' not in res.data
-        assert b'Mark all viewed' not in res.data
+        assert b'class="has-unviewed' not in res.data
         assert b'head title' not in res.data  # Should not be present because this is off by default
         assert b'test-endpoint' in res.data
 
@@ -133,7 +133,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
-    assert b'Mark all viewed' in res.data
+    assert b'class="has-unviewed' in res.data
 
     # It should have picked up the <title>
     assert b'head title' in res.data
@@ -144,7 +144,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # hit the mark all viewed link
     res = client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
 
-    assert b'Mark all viewed' not in res.data
+    assert b'class="has-unviewed' not in res.data
     assert b'unviewed' not in res.data
 
     # #2458 "clear history" should make the Watch object update its status correctly when the first snapshot lands again

changedetectionio/tests/test_basic_socketio.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+
+import time
+from flask import url_for
+from .util import (
+    set_original_response,
+    set_modified_response,
+    live_server_setup,
+    wait_for_all_checks
+)
+from loguru import logger
+
+def run_socketio_watch_update_test(client, live_server, password_mode=""):
+    """Test that the socketio emits a watch update event when content changes"""
+
+    # Set up the test server
+    set_original_response()
+
+    # Get the SocketIO instance from the app
+    from changedetectionio.flask_app import app
+    socketio = app.extensions['socketio']
+
+    # Create a test client for SocketIO
+    socketio_test_client = socketio.test_client(app, flask_test_client=client)
+    if password_mode == "not logged in, should exit on connect":
+        assert not socketio_test_client.is_connected(), "Failed to connect to Socket.IO server because it should bounce this connect"
+        return
+
+    assert socketio_test_client.is_connected(), "Failed to connect to Socket.IO server"
+    print("Successfully connected to Socket.IO server")
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("imports.import_page"),
+        data={"urls": url_for('test_endpoint', _external=True)},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    res = client.get(url_for("watchlist.index"))
+    assert url_for('test_endpoint', _external=True).encode() in res.data
+
+    # Wait for initial check to complete
+    wait_for_all_checks(client)
+
+    # Clear any initial messages
+    socketio_test_client.get_received()
+
+    # Make a change to trigger an update
+    set_modified_response()
+
+    # Force recheck
+    res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    assert b'Queued 1 watch for rechecking.' in res.data
+
+    # Wait for the watch to be checked
+    wait_for_all_checks(client)
+
+    has_watch_update = False
+    has_unviewed_update = False
+
+    for i in range(10):
+        # Get received events
+        received = socketio_test_client.get_received()
+
+        if received:
+            logger.info(f"Received {len(received)} events after {i+1} seconds")
+
+            # Check for watch_update events with unviewed=True
+            for event in received:
+                if event['name'] == 'watch_update':
+                    has_watch_update = True
+                    if event['args'][0]['watch'].get('unviewed', False):
+                        has_unviewed_update = True
+                        logger.info("Found unviewed update event!")
+                        break
+
+        if has_unviewed_update:
+            break
+
+        # Force a recheck every 5 seconds to ensure events are emitted
+#        if i > 0 and i % 5 == 0:
+#            print(f"Still waiting for events, forcing another recheck...")
+#            res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+#            assert b'Queued 1 watch for rechecking.' in res.data
+#            wait_for_all_checks(client)
+
+#        print(f"Waiting for unviewed update event... {i+1}/{max_wait}")
+        time.sleep(1)
+
+    # Verify we received watch_update events
+    assert has_watch_update, "No watch_update events received"
+
+    # Verify we received an unviewed event
+    assert has_unviewed_update, "No watch_update event with unviewed=True received"
+
+    # Alternatively, check directly if the watch in the datastore is marked as unviewed
+    from changedetectionio.flask_app import app
+    datastore = app.config.get('DATASTORE')
+
+    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+
+    # Get the watch from the datastore
+    watch = datastore.data['watching'].get(watch_uuid)
+    assert watch, f"Watch {watch_uuid} not found in datastore"
+    assert watch.has_unviewed, "The watch was not marked as unviewed after content change"
+
+    # Clean up
+    client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
+
+def test_everything(live_server, client):
+
+    live_server_setup(live_server)
+
+    run_socketio_watch_update_test(password_mode="", live_server=live_server, client=client)
+
+    ############################ Password required auth check ##############################
+
+    # Enable password check and diff page access bypass
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={"application-password": "foobar",
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    assert b"Password protection enabled." in res.data
+
+    run_socketio_watch_update_test(password_mode="not logged in, should exit on connect", live_server=live_server, client=client)
+    res = client.post(
+        url_for("login"),
+        data={"password": "foobar"},
+        follow_redirects=True
+    )
+
+    # Yes we are correctly logged in
+    assert b"LOG OUT" in res.data
+    run_socketio_watch_update_test(password_mode="should be like normal", live_server=live_server, client=client)

changedetectionio/tests/test_conditions.py

@@ -45,11 +45,15 @@ def set_number_out_of_range_response(number="150"):
         f.write(test_return_data)
 
 
+def test_setup(client, live_server):
+    """Test that both text and number conditions work together with AND logic."""
+    live_server_setup(live_server)
+
 def test_conditions_with_text_and_number(client, live_server):
     """Test that both text and number conditions work together with AND logic."""
     
     set_original_response("50")
-    live_server_setup(live_server)
+    #live_server_setup(live_server)
 
     test_url = url_for('test_endpoint', _external=True)
 
@@ -192,6 +196,150 @@ def test_condition_validate_rule_row(client, live_server):
     )
     assert res.status_code == 200
     assert b'false' in res.data
+    # cleanup for the next
+    client.get(
+        url_for("ui.form_delete", uuid="all"),
+        follow_redirects=True
+    )
 
 
 
+# If there was only a change in the whitespacing, then we shouldnt have a change detected
+def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage):
+    #live_server_setup(live_server)
+
+    test_return_data = """<html>
+       <body>
+     Some initial text<br>
+     <p>Which is across multiple lines</p>
+     <br>
+     So let's see what happens.  <br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("imports.import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Give the thread time to pick it up
+    wait_for_all_checks(client)
+
+    # Check it saved
+    res = client.get(
+        url_for("ui.ui_edit.edit_page", uuid="first"),
+    )
+
+    # Assert the word count is counted correctly
+    assert b'<td>13</td>' in res.data
+
+    # cleanup for the next
+    client.get(
+        url_for("ui.form_delete", uuid="all"),
+        follow_redirects=True
+    )
+
+# If there was only a change in the whitespacing, then we shouldnt have a change detected
+def test_lev_conditions_plugin(client, live_server, measure_memory_usage):
+    #live_server_setup(live_server)
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("""<html>
+       <body>
+     Some initial text<br>
+     <p>Which is across multiple lines</p>
+     <br>
+     So let's see what happens.  <br>
+     </body>
+     </html>
+    """)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("ui.ui_views.form_quick_watch_add"),
+        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        follow_redirects=True
+    )
+    assert b"Watch added in Paused state, saving will unpause" in res.data
+
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    # Give the thread time to pick it up
+    wait_for_all_checks(client)
+    res = client.post(
+        url_for("ui.ui_edit.edit_page", uuid=uuid, unpause_on_save=1),
+        data={
+            "url": test_url,
+            "fetch_backend": "html_requests",
+            "conditions_match_logic": "ALL",  # ALL = AND logic
+            "conditions-0-field": "levenshtein_ratio",
+            "conditions-0-operator": "<",
+            "conditions-0-value": "0.8" # needs to be more of a diff to trigger a change
+        },
+        follow_redirects=True
+    )
+
+    assert b"unpaused" in res.data
+
+    wait_for_all_checks(client)
+    res = client.get(url_for("watchlist.index"))
+    assert b'unviewed' not in res.data
+
+    # Check the content saved initially, even tho a condition was set - this is the first snapshot so shouldnt be affected by conditions
+    res = client.get(
+        url_for("ui.ui_views.preview_page", uuid=uuid),
+        follow_redirects=True
+    )
+    assert b'Which is across multiple lines' in res.data
+
+
+    ############### Now change it a LITTLE bit...
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("""<html>
+       <body>
+     Some initial text<br>
+     <p>Which is across multiple lines</p>
+     <br>
+     So let's see what happenxxxxxxxxx.  <br>
+     </body>
+     </html>
+    """)
+
+    res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    assert b'Queued 1 watch for rechecking.' in res.data
+    wait_for_all_checks(client)
+
+    res = client.get(url_for("watchlist.index"))
+    assert b'unviewed' not in res.data #because this will be like 0.90 not 0.8 threshold
+
+    ############### Now change it a MORE THAN 50%
+    test_return_data = """<html>
+       <body>
+     Some sxxxx<br>
+     <p>Which is across a lines</p>
+     <br>
+     ok.  <br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    assert b'Queued 1 watch for rechecking.' in res.data
+    wait_for_all_checks(client)
+    res = client.get(url_for("watchlist.index"))
+    assert b'unviewed' in res.data
+    # cleanup for the next
+    client.get(
+        url_for("ui.form_delete", uuid="all"),
+        follow_redirects=True
+    )

changedetectionio/tests/test_history_consistency.py

@@ -10,8 +10,8 @@ from urllib.parse import urlparse, parse_qs
 
 def test_consistent_history(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
-
-    r = range(1, 30)
+    workers = int(os.getenv("FETCH_WORKERS", 10))
+    r = range(1, 10+workers)
 
     for one in r:
         test_url = url_for('test_endpoint', content_type="text/html", content=str(one), _external=True)
@@ -46,9 +46,10 @@ def test_consistent_history(client, live_server, measure_memory_usage):
 
     # assert the right amount of watches was found in the JSON
     assert len(json_obj['watching']) == len(r), "Correct number of watches was found in the JSON"
-
+    i=0
     # each one should have a history.txt containing just one line
     for w in json_obj['watching'].keys():
+        i+=1
         history_txt_index_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, 'history.txt')
         assert os.path.isfile(history_txt_index_file), f"History.txt should exist where I expect it at {history_txt_index_file}"
 
@@ -58,8 +59,8 @@ def test_consistent_history(client, live_server, measure_memory_usage):
             assert len(tmp_history) == 1, "History.txt should contain 1 line"
 
         # Should be two files,. the history.txt , and the snapshot.txt
-        files_in_watch_dir = os.listdir(os.path.join(live_server.app.config['DATASTORE'].datastore_path,
-                                                     w))
+        files_in_watch_dir = os.listdir(os.path.join(live_server.app.config['DATASTORE'].datastore_path, w))
+
         # Find the snapshot one
         for fname in files_in_watch_dir:
             if fname != 'history.txt' and 'html' not in fname:
@@ -75,7 +76,6 @@ def test_consistent_history(client, live_server, measure_memory_usage):
 
         assert len(files_in_watch_dir) == 3, "Should be just three files in the dir, html.br snapshot, history.txt and the extracted text snapshot"
 
-
     json_db_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, 'url-watches.json')
     with open(json_db_file, 'r') as f:
         assert '"default"' not in f.read(), "'default' probably shouldnt be here, it came from when the 'default' Watch vars were accidently being saved"

changedetectionio/tests/test_ignore_text.py

@@ -32,13 +32,14 @@ def test_strip_text_func():
     stripped_content = html_tools.strip_ignore_text(test_content, ignore)
     assert stripped_content == "Some initial text\n\nWhich is across multiple lines\n\n\n\nSo let's see what happens."
 
-def set_original_ignore_response():
-    test_return_data = """<html>
+def set_original_ignore_response(ver_stamp="123"):
+    test_return_data = f"""<html>
        <body>
      Some initial text<br>
      <p>Which is across multiple lines</p>
      <br>
      So let's see what happens.  <br>
+     <link href="https://www.somesite/wp-content/themes/cooltheme/style2.css?v={ver_stamp}" rel="stylesheet"/>
      </body>
      </html>
 
@@ -48,13 +49,14 @@ def set_original_ignore_response():
         f.write(test_return_data)
 
 
-def set_modified_original_ignore_response():
-    test_return_data = """<html>
+def set_modified_original_ignore_response(ver_stamp="123"):
+    test_return_data = f"""<html>
        <body>
      Some NEW nice initial text<br>
      <p>Which is across multiple lines</p>
      <br>
      So let's see what happens.  <br>
+     <link href="https://www.somesite/wp-content/themes/cooltheme/style2.css?v={ver_stamp}" rel="stylesheet"/>
      <p>new ignore stuff</p>
      <p>blah</p>
      </body>
@@ -67,14 +69,15 @@ def set_modified_original_ignore_response():
 
 
 # Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
-def set_modified_ignore_response():
-    test_return_data = """<html>
+def set_modified_ignore_response(ver_stamp="123"):
+    test_return_data = f"""<html>
        <body>
      Some initial text<br>
      <p>Which is across multiple lines</p>
      <P>ZZZZz</P>
      <br>
      So let's see what happens.  <br>
+     <link href="https://www.somesite/wp-content/themes/cooltheme/style2.css?v={ver_stamp}" rel="stylesheet"/>
      </body>
      </html>
 
@@ -165,9 +168,9 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     assert b'Deleted' in res.data
 
 # When adding some ignore text, it should not trigger a change, even if something else on that line changes
-def test_check_global_ignore_text_functionality(client, live_server, measure_memory_usage):
-    #live_server_setup(live_server)
-    ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"
+def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
+    ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ\r\n"+extra_ignore
+
     set_original_ignore_response()
 
     # Goto the settings page, add our ignore text
@@ -186,6 +189,10 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
+    if as_source:
+        # Switch to source mode so we can test that too!
+        test_url = "source:"+test_url
+
     res = client.post(
         url_for("imports.import_page"),
         data={"urls": test_url},
@@ -203,12 +210,15 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-
+    wait_for_all_checks(client)
     # Check it saved
     res = client.get(
         url_for("settings.settings_page"),
     )
-    assert bytes(ignore_text.encode('utf-8')) in res.data
+
+    for i in ignore_text.splitlines():
+        assert bytes(i.encode('utf-8')) in res.data
+
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -221,7 +231,8 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem
 
     # Make a change which includes the ignore text, it should be ignored and no 'change' triggered
     # It adds text with "ZZZZzzzz" and "ZZZZ" is in the ignore list
-    set_modified_ignore_response()
+    # And tweaks the ver_stamp which should be picked up by global regex ignore
+    set_modified_ignore_response(ver_stamp=time.time())
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -243,3 +254,11 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
+
+def test_check_global_ignore_text_functionality(client, live_server):
+    #live_server_setup(live_server)
+    _run_test_global_ignore(client, as_source=False)
+
+def test_check_global_ignore_text_functionality_as_source(client, live_server):
+    #live_server_setup(live_server)
+    _run_test_global_ignore(client, as_source=True, extra_ignore='/\?v=\d/')

changedetectionio/tests/test_ignorehyperlinks.py

@@ -3,7 +3,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 
 
 def test_setup(live_server):
@@ -70,19 +70,18 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
     # set a new html text with a modified link
     set_modified_ignore_response()
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # We should not see the rendered anchor tag
     res = client.get(url_for("ui.ui_views.preview_page", uuid="first"))
@@ -104,7 +103,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
 
 

changedetectionio/tests/test_live_preview.py

@@ -6,7 +6,7 @@ from changedetectionio.tests.util import live_server_setup, wait_for_all_checks,
 
 def set_response():
 
-    data = f"""<html>
+    data = """<html>
        <body>Awesome, you made it<br>
 yeah the socks request worked<br>
 something to ignore<br>

changedetectionio/tests/test_nonrenderable_pages.py

@@ -100,7 +100,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # A totally zero byte (#2528) response should also not trigger an error
     set_zero_byte_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-
+    wait_for_all_checks(client)
     # 2877
     assert watch.last_changed == watch['last_checked']
 

changedetectionio/tests/test_request.py

@@ -424,3 +424,27 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     # unlink headers.txt on start/stop
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
+
+def test_headers_validation(client, live_server):
+    #live_server_setup(live_server)
+
+    test_url = url_for('test_headers', _external=True)
+    res = client.post(
+        url_for("imports.import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    res = client.post(
+        url_for("ui.ui_edit.edit_page", uuid="first"),
+        data={
+            "url": test_url,
+            "fetch_backend": 'html_requests',
+            "headers": "User-AGent agent-from-watch\r\nsadfsadfsadfsdaf\r\n:foobar"},
+        follow_redirects=True
+    )
+
+    assert b"Line 1 is missing a ':' separator." in res.data
+    assert b"Line 3 has an empty key." in res.data
+

changedetectionio/tests/test_rss.py

@@ -219,6 +219,15 @@ def test_rss_bad_chars_breaking(client, live_server):
     rss_token = extract_rss_token_from_UI(client)
 
     uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    i=0
+    from loguru import logger
+    # Because chardet could take a long time
+    while i<=10:
+        logger.debug(f"History was {live_server.app.config['DATASTORE'].data['watching'][uuid].history_n}..")
+        if live_server.app.config['DATASTORE'].data['watching'][uuid].history_n ==2:
+            break
+            i+=1
+        time.sleep(2)
     assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n == 2
 
     # Check RSS feed is still working

changedetectionio/tests/util.py

@@ -126,18 +126,50 @@ def extract_UUID_from_client(client):
     uuid = m.group(1)
     return uuid.strip()
 
-def wait_for_all_checks(client):
-    # actually this is not entirely true, it can still be 'processing' but not in the queue
-    # Loop waiting until done..
-    attempt=0
-    # because sub-second rechecks are problematic in testing, use lots of delays
-    time.sleep(1)
-    while attempt < 60:
-        res = client.get(url_for("watchlist.index"))
-        if not b'Checking now' in res.data:
-            break
-        logging.getLogger().info("Waiting for watch-list to not say 'Checking now'.. {}".format(attempt))
-        time.sleep(1)
+
+def wait_for_all_checks(client=None):
+    """
+    Waits until the queue is empty and remains empty for at least `required_empty_duration` seconds,
+    and also ensures no running threads have `current_uuid` set.
+    Retries for up to `max_attempts` times, sleeping `wait_between_attempts` seconds between checks.
+    """
+    from changedetectionio.flask_app import update_q as global_update_q, running_update_threads
+
+    # Configuration
+    attempt = 0
+    i=0
+    max_attempts = 60
+    required_empty_duration = 0.2
+
+    logger = logging.getLogger()
+    time.sleep(1.2)
+
+    empty_since = None
+
+    while attempt < max_attempts:
+        q_length = global_update_q.qsize()
+
+        # Check if any threads are still processing
+        time.sleep(1.2)
+        any_threads_busy = any(t.current_uuid for t in running_update_threads)
+
+
+        if q_length == 0 and not any_threads_busy:
+            if empty_since is None:
+                empty_since = time.time()
+                logger.info(f"Queue empty and no active threads at attempt {attempt}, starting empty timer...")
+            elif time.time() - empty_since >= required_empty_duration:
+                logger.info(f"Queue has been empty and threads idle for {required_empty_duration} seconds. Done waiting.")
+                break
+            else:
+                logger.info(f"Still waiting: queue empty and no active threads, but not yet {required_empty_duration} seconds...")
+        else:
+            if q_length != 0:
+                logger.info(f"Queue not empty (size={q_length}), resetting timer.")
+            if any_threads_busy:
+                busy_threads = [t.name for t in running_update_threads if t.current_uuid]
+                logger.info(f"Threads still busy: {busy_threads}, resetting timer.")
+            empty_since = None
         attempt += 1
 
     time.sleep(1)
@@ -296,6 +328,8 @@ def live_server_setup(live_server):
 
     live_server.start()
 
+
+
 def get_index(client):
     import inspect
     # Get the caller's frame (parent function)

changedetectionio/update_worker.py

@@ -2,6 +2,7 @@ from .processors.exceptions import ProcessorException
 import changedetectionio.content_fetchers.exceptions as content_fetchers_exceptions
 from changedetectionio.processors.text_json_diff.processor import FilterNotFoundInResponse
 from changedetectionio import html_tools
+from changedetectionio.flask_app import watch_check_update
 
 import importlib
 import os
@@ -242,17 +243,16 @@ class update_worker(threading.Thread):
                 os.unlink(full_path)
 
     def run(self):
-        
+
         while not self.app.config.exit.is_set():
             update_handler = None
+            watch = None
 
             try:
                 queued_item_data = self.q.get(block=False)
             except queue.Empty:
                 pass
-
             else:
-
                 uuid = queued_item_data.item.get('uuid')
                 fetch_start_time = round(time.time())  # Also used for a unique history key for now
                 self.current_uuid = uuid
@@ -272,6 +272,8 @@ class update_worker(threading.Thread):
                     logger.info(f"Processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
 
                     try:
+                        watch_check_update.send(watch_uuid=uuid)
+
                         # Processor is what we are using for detecting the "Change"
                         processor = watch.get('processor', 'text_json_diff')
 
@@ -588,12 +590,18 @@ class update_worker(threading.Thread):
                                                                        'check_count': count
                                                                        })
 
-
                 self.current_uuid = None  # Done
                 self.q.task_done()
+
+                # Send signal for watch check completion with the watch data
+                if watch:
+                    logger.info(f"Sending watch_check_update signal for UUID {watch['uuid']}")
+                    watch_check_update.send(watch_uuid=watch['uuid'])
+
                 update_handler = None
                 logger.debug(f"Watch {uuid} done in {time.time()-fetch_start_time:.2f}s")
 
+
                 # Give the CPU time to interrupt
                 time.sleep(0.1)
 

docker-compose.yml

@@ -72,7 +72,7 @@ services:
   
       # Comment out ports: when using behind a reverse proxy , enable networks: etc.
       ports:
-        - 5000:5000
+        - 127.0.0.1:5000:5000
       restart: unless-stopped
 
      # Used for fetching pages via WebDriver+Chrome where you need Javascript support.
@@ -82,7 +82,7 @@ services:
      # If WEBDRIVER or PLAYWRIGHT are enabled, changedetection container depends on that
      # and must wait before starting (substitute "browser-chrome" with "playwright-chrome" if last one is used)
 #      depends_on:
-#          sockpuppetbrowser:
+#          browser-sockpuppet-chrome:
 #              condition: service_started
 
 

requirements.txt

@@ -9,6 +9,9 @@ flask_restful
 flask_cors # For the Chrome extension to operate
 flask_wtf~=1.2
 flask~=2.3
+flask-socketio>=5.5.1
+python-socketio>=5.13.0
+python-engineio>=4.12.0
 inscriptis~=2.2
 pytz
 timeago~=1.0
@@ -32,7 +35,7 @@ dnspython==2.6.1 # related to eventlet fixes
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise==1.9.2
+apprise==1.9.3
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 # use any version other than 2.0.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
@@ -42,7 +45,7 @@ paho-mqtt!=2.0.*
 cryptography~=42.0.8
 
 # Used for CSS filtering
-beautifulsoup4
+beautifulsoup4>=4.0.0
 
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
 # #2328 - 5.2.0 and 5.2.1 had extra CPU flag CFLAGS set which was not compatible on older hardware
@@ -53,7 +56,7 @@ lxml >=4.8.0,<6,!=5.2.0,!=5.2.1
 # XPath 2.0-3.1 support - 4.2.0 broke something?
 elementpath==4.1.5
 
-selenium~=4.14.0
+selenium~=4.31.0
 
 # https://github.com/pallets/werkzeug/issues/2985
 # Maybe related to pytest?
@@ -70,7 +73,7 @@ jq~=1.3; python_version >= "3.8" and sys_platform == "linux"
 
 # playwright is installed at Dockerfile build time because it's not available on all platforms
 
-pyppeteer-ng==2.0.0rc9
+pyppeteer-ng==2.0.0rc10
 
 pyppeteerstealth>=0.0.4
 
@@ -90,9 +93,14 @@ extruct
 # For cleaning up unknown currency formats
 babel
 
+levenshtein
+
 # Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096
 greenlet >= 3.0.3
 
+# Used for realtime socketio mode (so its a different driver to eventlet/threading not to interfere with playwright)
+gevent
+
 # Pinned or it causes problems with flask_expects_json which seems unmaintained
 referencing==0.35.1
 
@@ -101,6 +109,8 @@ panzi-json-logic
 # For conditions - extracted number from a body of text
 price-parser
 
+# flask_socket_io - incorrect package name, already have flask-socketio above
+
 # Scheduler - Windows seemed to miss a lot of default timezone info (even "UTC" !)
 tzdata
 
@@ -110,3 +120,9 @@ pluggy ~= 1.5
 
 # Needed for testing, cross-platform for process and system monitoring
 psutil==7.0.0
+
+ruff >= 0.11.2
+pre_commit >= 4.2.0
+
+# For events between checking and socketio updates
+blinker