refactor attempt

.github/workflows/containers.yml

@@ -103,14 +103,6 @@ jobs:
             ghcr.io/${{ github.repository }}
           tags: |
             type=raw,value=dev
-          labels: |
-            org.opencontainers.image.created=${{ github.event.release.published_at }}
-            org.opencontainers.image.description=Website, webpage change detection, monitoring and notifications.
-            org.opencontainers.image.documentation=https://changedetection.io
-            org.opencontainers.image.revision=${{ github.sha }}
-            org.opencontainers.image.source=https://github.com/dgtlmoon/changedetection.io
-            org.opencontainers.image.title=changedetection.io
-            org.opencontainers.image.url=https://changedetection.io
 
       - name: Build and push :dev
         id: docker_build
@@ -136,7 +128,7 @@ jobs:
           echo "Release tag: ${{ github.event.release.tag_name }}"
           echo "Github ref: ${{ github.ref }}"
           echo "Github ref name: ${{ github.ref_name }}"
-
+          
       - name: Docker meta :tag
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
         uses: docker/metadata-action@v6
@@ -150,15 +142,6 @@ jobs:
                 type=semver,pattern={{major}}.{{minor}},value=${{ github.event.release.tag_name }}
                 type=semver,pattern={{major}},value=${{ github.event.release.tag_name }}
                 type=raw,value=latest
-            labels: |
-              org.opencontainers.image.created=${{ github.event.release.published_at }}
-              org.opencontainers.image.description=Website, webpage change detection, monitoring and notifications.
-              org.opencontainers.image.documentation=https://changedetection.io
-              org.opencontainers.image.revision=${{ github.sha }}
-              org.opencontainers.image.source=https://github.com/dgtlmoon/changedetection.io
-              org.opencontainers.image.title=changedetection.io
-              org.opencontainers.image.url=https://changedetection.io
-              org.opencontainers.image.version=${{ github.event.release.tag_name }}
 
       - name: Build and push :tag
         id: docker_build_tag_release

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 # Semver means never use .01, or 00. Should be .1.
-__version__ = '0.54.6'
+__version__ = '0.54.4'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/api/Watch.py

@@ -338,7 +338,7 @@ class WatchHistoryDiff(Resource):
             word_diff = True
 
         # Get boolean diff preferences with defaults from DIFF_PREFERENCES_CONFIG
-        changes_only = strtobool(request.args.get('changesOnly', 'false'))
+        changes_only = strtobool(request.args.get('changesOnly', 'true'))
         ignore_whitespace = strtobool(request.args.get('ignoreWhitespace', 'false'))
         include_removed = strtobool(request.args.get('removed', 'true'))
         include_added = strtobool(request.args.get('added', 'true'))
@@ -349,7 +349,7 @@ class WatchHistoryDiff(Resource):
             previous_version_file_contents=from_version_file_contents,
             newest_version_file_contents=to_version_file_contents,
             ignore_junk=ignore_whitespace,
-            include_equal=not changes_only,
+            include_equal=changes_only,
             include_removed=include_removed,
             include_added=include_added,
             include_replaced=include_replaced,
@@ -567,4 +567,4 @@ class CreateWatch(Resource):
 
                 return {'status': f'OK, queueing {len(watches_to_queue)} watches in background'}, 202
 
-        return list, 200
+        return list, 200

changedetectionio/blueprint/browser_steps/__init__.py

@@ -102,35 +102,6 @@ def run_async_in_browser_loop(coro):
     else:
         raise RuntimeError("Browser steps event loop is not available")
 
-async def _close_session_resources(session_data, label=''):
-    """Close all browser resources for a session in the correct order.
-
-    browserstepper.cleanup() closes page+context but not the browser itself.
-    For CloakBrowser, browser.close() is what stops the local Chromium process via pw.stop().
-    For the default CDP path, playwright_context.stop() shuts down the playwright instance.
-    """
-    browserstepper = session_data.get('browserstepper')
-    if browserstepper:
-        try:
-            await browserstepper.cleanup()
-        except Exception as e:
-            logger.error(f"Error cleaning up browserstepper{label}: {e}")
-
-    browser = session_data.get('browser')
-    if browser:
-        try:
-            await asyncio.wait_for(browser.close(), timeout=5.0)
-        except Exception as e:
-            logger.warning(f"Error closing browser{label}: {e}")
-
-    playwright_context = session_data.get('playwright_context')
-    if playwright_context:
-        try:
-            await playwright_context.stop()
-        except Exception as e:
-            logger.warning(f"Error stopping playwright context{label}: {e}")
-
-
 def cleanup_expired_sessions():
     """Remove expired browsersteps sessions and cleanup their resources"""
     global browsersteps_sessions, browsersteps_watch_to_session
@@ -148,10 +119,13 @@ def cleanup_expired_sessions():
         logger.debug(f"Cleaning up expired browsersteps session {session_id}")
         session_data = browsersteps_sessions[session_id]
 
-        try:
-            run_async_in_browser_loop(_close_session_resources(session_data, label=f" for session {session_id}"))
-        except Exception as e:
-            logger.error(f"Error cleaning up session {session_id}: {e}")
+        # Cleanup playwright resources asynchronously
+        browserstepper = session_data.get('browserstepper')
+        if browserstepper:
+            try:
+                run_async_in_browser_loop(browserstepper.cleanup())
+            except Exception as e:
+                logger.error(f"Error cleaning up session {session_id}: {e}")
 
         # Remove from sessions dict
         del browsersteps_sessions[session_id]
@@ -178,10 +152,12 @@ def cleanup_session_for_watch(watch_uuid):
 
     session_data = browsersteps_sessions.get(session_id)
     if session_data:
-        try:
-            run_async_in_browser_loop(_close_session_resources(session_data, label=f" for watch {watch_uuid}"))
-        except Exception as e:
-            logger.error(f"Error cleaning up session {session_id} for watch {watch_uuid}: {e}")
+        browserstepper = session_data.get('browserstepper')
+        if browserstepper:
+            try:
+                run_async_in_browser_loop(browserstepper.cleanup())
+            except Exception as e:
+                logger.error(f"Error cleaning up session {session_id} for watch {watch_uuid}: {e}")
 
         # Remove from sessions dict
         del browsersteps_sessions[session_id]
@@ -202,69 +178,59 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         import time
         from playwright.async_api import async_playwright
 
+        # We keep the playwright session open for many minutes
         keepalive_seconds = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
-        keepalive_ms = ((keepalive_seconds + 3) * 1000)
 
         browsersteps_start_session = {'start_time': time.time()}
 
-        # Build proxy dict first — needed by both the CDP path and fetcher-specific launchers
-        proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
-        proxy = None
-        if proxy_id:
-            proxy_url = datastore.proxy_list.get(proxy_id, {}).get('url')
-            if proxy_url:
-                from urllib.parse import urlparse
-                parsed = urlparse(proxy_url)
-                proxy = {'server': proxy_url}
-                if parsed.username:
-                    proxy['username'] = parsed.username
-                if parsed.password:
-                    proxy['password'] = parsed.password
-                logger.debug(f"Browser Steps: UUID {watch_uuid} selected proxy {proxy_url}")
+        # Create a new async playwright instance for browser steps
+        playwright_instance = async_playwright()
+        playwright_context = await playwright_instance.start()
 
-        # Resolve the fetcher class for this watch so we can ask it to launch its own browser
-        # if it supports that (e.g. CloakBrowser, which runs locally rather than via CDP)
-        watch = datastore.data['watching'][watch_uuid]
-        from changedetectionio import content_fetchers
-        fetcher_name = watch.get_fetch_backend or 'system'
-        if fetcher_name == 'system':
-            fetcher_name = datastore.data['settings']['application'].get('fetch_backend', 'html_requests')
-        fetcher_class = getattr(content_fetchers, fetcher_name, None)
-
-        browser = None
-        playwright_context = None
-
-        # If the fetcher has its own browser launch for the live steps UI, use it.
-        # get_browsersteps_browser(proxy, keepalive_ms) returns (browser, playwright_context_or_None)
-        # or None to fall back to the default CDP path.
-        if fetcher_class and hasattr(fetcher_class, 'get_browsersteps_browser'):
-            result = await fetcher_class.get_browsersteps_browser(proxy=proxy, keepalive_ms=keepalive_ms)
-            if result is not None:
-                browser, playwright_context = result
-                logger.debug(f"Browser Steps: using fetcher-specific browser for '{fetcher_name}'")
-
-        # Default: connect to the remote Playwright/sockpuppetbrowser via CDP
-        if browser is None:
-            playwright_instance = async_playwright()
-            playwright_context = await playwright_instance.start()
-            base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '').strip('"')
-            a = "?" if '?' not in base_url else '&'
-            base_url += a + f"timeout={keepalive_ms}"
-            browser = await playwright_context.chromium.connect_over_cdp(base_url, timeout=keepalive_ms)
-            logger.debug(f"Browser Steps: using CDP connection to {base_url}")
+        keepalive_ms = ((keepalive_seconds + 3) * 1000)
+        base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '').strip('"')
+        a = "?" if not '?' in base_url else '&'
+        base_url += a + f"timeout={keepalive_ms}"
 
+        browser = await playwright_context.chromium.connect_over_cdp(base_url, timeout=keepalive_ms)
         browsersteps_start_session['browser'] = browser
         browsersteps_start_session['playwright_context'] = playwright_context
 
+        proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
+        proxy = None
+        if proxy_id:
+            proxy_url = datastore.proxy_list.get(proxy_id).get('url')
+            if proxy_url:
+
+                # Playwright needs separate username and password values
+                from urllib.parse import urlparse
+                parsed = urlparse(proxy_url)
+                proxy = {'server': proxy_url}
+
+                if parsed.username:
+                    proxy['username'] = parsed.username
+
+                if parsed.password:
+                    proxy['password'] = parsed.password
+
+                logger.debug(f"Browser Steps: UUID {watch_uuid} selected proxy {proxy_url}")
+
+        # Tell Playwright to connect to Chrome and setup a new session via our stepper interface
         browserstepper = browser_steps.browsersteps_live_ui(
             playwright_browser=browser,
             proxy=proxy,
-            start_url=watch.link,
-            headers=watch.get('headers')
+            start_url=datastore.data['watching'][watch_uuid].link,
+            headers=datastore.data['watching'][watch_uuid].get('headers')
         )
+        
+        # Initialize the async connection
         await browserstepper.connect(proxy=proxy)
+        
         browsersteps_start_session['browserstepper'] = browserstepper
 
+        # For test
+        #await browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time()))
+
         return browsersteps_start_session
 
 

changedetectionio/blueprint/settings/templates/settings.html

@@ -154,8 +154,9 @@
                     </span>
                 </div>
                 <div class="pure-control-group">
-                <br>
-                    {{ _('Tip:') }} <a href="{{ url_for('settings.settings_page')}}#proxies">{{ _('Connect using Bright Data proxies, find out more here.') }}</a>
+                                        <br>
+                    {{ _('Tip:') }} <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">{{ _('Connect using Bright Data and Oxylabs Proxies, find out more here.') }}</a>
+
                 </div>
             </div>
 
@@ -351,7 +352,7 @@ nav
                     </div>
                 </div>
 
-                <p><strong>{{ _('Tip') }}</strong>: {{ _('"Residential" and "Mobile" proxy type can be more successful than "Data Center" for blocked websites.') }}</p>
+                <p><strong>{{ _('Tip') }}</strong>: {{ _('"Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.') }}</p>
 
                 <div class="pure-control-group" id="extra-proxies-setting">
                 {{ render_fieldlist_with_inline_errors(form.requests.form.extra_proxies) }}

changedetectionio/blueprint/ui/preview.py

@@ -60,8 +60,12 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         versions = []
         timestamp = None
 
+        system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
         extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
-        is_html_webdriver = watch.fetcher_supports_screenshots
+
+        is_html_webdriver = False
+        if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
+            is_html_webdriver = True
 
         triggered_line_numbers = []
         ignored_line_numbers = []

changedetectionio/html_tools.py

@@ -487,25 +487,13 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
         except json.JSONDecodeError as e:
             logger.warning(f"Error processing JSON {content[:20]}...{str(e)})")
     else:
-        # Check for JSONP wrapper: someCallback({...}) or some.namespace({...})
-        # Server may claim application/json but actually return JSONP
-        jsonp_match = re.match(r'^\w[\w.]*\s*\((.+)\)\s*;?\s*$', content.lstrip("\ufeff").strip(), re.DOTALL)
-        if jsonp_match:
-            try:
-                inner = jsonp_match.group(1).strip()
-                logger.warning(f"Content looks like JSONP, attempting to extract inner JSON for filter '{json_filter}'")
-                stripped_text_from_html = _parse_json(json.loads(inner), json_filter)
-            except json.JSONDecodeError as e:
-                logger.warning(f"Error processing JSONP inner content {content[:20]}...{str(e)})")
-
-        if not stripped_text_from_html:
-            # Probably something else, go fish inside for it
-            try:
-                stripped_text_from_html = extract_json_blob_from_html(content=content,
-                                                                      ensure_is_ldjson_info_type=ensure_is_ldjson_info_type,
-                                                                      json_filter=json_filter)
-            except json.JSONDecodeError as e:
-                logger.warning(f"Error processing JSON while extracting JSON from HTML blob {content[:20]}...{str(e)})")
+        # Probably something else, go fish inside for it
+        try:
+            stripped_text_from_html = extract_json_blob_from_html(content=content,
+                                                                  ensure_is_ldjson_info_type=ensure_is_ldjson_info_type,
+                                                                  json_filter=json_filter                                                                  )
+        except json.JSONDecodeError as e:
+            logger.warning(f"Error processing JSON while extracting JSON from HTML blob {content[:20]}...{str(e)})")
 
     if not stripped_text_from_html:
         # Re 265 - Just return an empty string when filter not found

changedetectionio/model/Watch.py

@@ -388,25 +388,6 @@ class model(EntityPersistenceMixin, watch_base):
 
         return self.get('fetch_backend')
 
-    @property
-    def fetcher_supports_screenshots(self):
-        """Return True if the fetcher configured for this watch supports screenshots.
-
-        Resolves 'system' via self._datastore, then checks supports_screenshots on
-        the actual fetcher class. Works for built-in and plugin fetchers alike.
-        """
-        from changedetectionio import content_fetchers
-
-        fetcher_name = self.get_fetch_backend  # already handles is_pdf → html_requests
-        if not fetcher_name or fetcher_name == 'system':
-            fetcher_name = self._datastore['settings']['application'].get('fetch_backend', 'html_requests')
-
-        fetcher_class = getattr(content_fetchers, fetcher_name, None)
-        if fetcher_class is None:
-            return False
-
-        return bool(getattr(fetcher_class, 'supports_screenshots', False))
-
     @property
     def is_pdf(self):
         url = str(self.get("url") or "").lower()

changedetectionio/notification_service.py

@@ -6,6 +6,7 @@ Extracted from update_worker.py to provide standalone notification functionality
 for both sync and async workers
 """
 import datetime
+from copy import deepcopy
 
 import pytz
 from loguru import logger
@@ -352,7 +353,7 @@ class NotificationService:
         """
         Send notification when content changes are detected
         """
-        n_object = NotificationContextData()
+
         watch = self.datastore.data['watching'].get(watch_uuid)
         if not watch:
             return
@@ -369,21 +370,51 @@ class NotificationService:
         # Should be a better parent getter in the model object
 
         # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
-        # this change probably not needed?
-        n_object['notification_urls'] = _check_cascading_vars(self.datastore, 'notification_urls', watch)
+        # If the watch has no notification_body for example, it will try to get from the first matching group or system setting
+
+        # Should be, if none in the watch, and no group tag ones found, then use system ones at the end
+        #n_object['notification_urls'] = _check_cascading_vars(self.datastore, 'notification_urls', watch)
+        n_object = NotificationContextData()
         n_object['notification_title'] = _check_cascading_vars(self.datastore,'notification_title', watch)
         n_object['notification_body'] = _check_cascading_vars(self.datastore,'notification_body', watch)
         n_object['notification_format'] = _check_cascading_vars(self.datastore,'notification_format', watch)
 
+        notification_objects = []
+        if n_object.get('notification_urls'):
+            notification_objects.append(n_object)
+
+
+        # LOGIC SHOULD BE something that all tests currently pass too
+        # !!! _check_cascading_vars is not really used much, only used here..
+        #
+
+
+        # If any related group/tag has a notification_url set, then we fan out horizontally and collect it as extra notifications
+        tags = self.datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))
+        logger.debug(f'{len(tags)} related to this watch')
+        if tags:
+            for tag_uuid, tag in tags.items():
+                logger.debug(f"Checking group/tag for notification URLs '{tag['title']}' Muted? '{tag.get('notification_muted')}', URLs {tag.get('notification_urls')}")
+                v = tag.get('notification_urls')
+                if v and not tag.get('notification_muted'):
+                    logger.debug("OK MAN")
+                    next_n_object = deepcopy(n_object)
+                    next_n_object['notification_urls'] = v
+                    next_n_object['notification_title'] = _check_cascading_vars(self.datastore, 'notification_title', watch)
+                    next_n_object['notification_body'] = _check_cascading_vars(self.datastore, 'notification_body', watch)
+                    next_n_object['notification_format'] = _check_cascading_vars(self.datastore, 'notification_format', watch)
+                    notification_objects.append(next_n_object)
+                    logger.debug(f"Adding notification from group/tag {tag['title']}")
+
+
         # (Individual watch) Only prepare to notify if the rules above matched
         queued = False
-        if n_object and n_object.get('notification_urls'):
+        if notification_objects:
             queued = True
-
             count = watch.get('notification_alert_count', 0) + 1
             self.datastore.update_watch(uuid=watch_uuid, update_obj={'notification_alert_count': count})
-
-            self.queue_notification_for_watch(n_object=n_object, watch=watch)
+            for n_object in notification_objects:
+                self.queue_notification_for_watch(n_object=n_object, watch=watch)
 
         return queued
 

changedetectionio/processors/extract.py

@@ -42,7 +42,10 @@ def render_form(watch, datastore, request, url_for, render_template, flash, redi
     # Get error information for the template
     screenshot_url = watch.get_screenshot()
 
-    is_html_webdriver = watch.fetcher_supports_screenshots
+    system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+    is_html_webdriver = False
+    if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
+        is_html_webdriver = True
 
     password_enabled_and_share_is_off = False
     if datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False):

changedetectionio/processors/magic.py

@@ -100,13 +100,7 @@ class guess_stream_type():
         if any(s in http_content_header for s in RSS_XML_CONTENT_TYPES):
             self.is_rss = True
         elif any(s in http_content_header for s in JSON_CONTENT_TYPES):
-            # JSONP detection: server claims application/json but content is actually JSONP (e.g. cb({...}))
-            # A JSONP response starts with an identifier followed by '(' - not valid JSON
-            if re.match(r'^\w[\w.]*\s*\(', test_content):
-                logger.warning(f"Content-Type header claims JSON but content looks like JSONP (starts with identifier+parenthesis) - treating as plaintext")
-                self.is_plaintext = True
-            else:
-                self.is_json = True
+            self.is_json = True
         elif 'pdf' in magic_content_header:
             self.is_pdf = True
         # magic will call a rss document 'xml'

changedetectionio/processors/text_json_diff/difference.py

@@ -154,7 +154,11 @@ def render(watch, datastore, request, url_for, render_template, flash, redirect,
 
     screenshot_url = watch.get_screenshot()
 
-    is_html_webdriver = watch.fetcher_supports_screenshots
+    system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+
+    is_html_webdriver = False
+    if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
+        is_html_webdriver = True
 
     password_enabled_and_share_is_off = False
     if datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False):

changedetectionio/realtime/events.py

@@ -29,11 +29,9 @@ def register_watch_operation_handlers(socketio, datastore):
             # Perform the operation
             if op == 'pause':
                 watch.toggle_pause()
-                watch.commit()
                 logger.info(f"Socket.IO: Toggled pause for watch {uuid}")
             elif op == 'mute':
                 watch.toggle_mute()
-                watch.commit()
                 logger.info(f"Socket.IO: Toggled mute for watch {uuid}")
             elif op == 'recheck':
                 # Import here to avoid circular imports

changedetectionio/realtime/socket_server.py

@@ -199,31 +199,8 @@ def handle_watch_update(socketio, **kwargs):
         logger.error(f"Socket.IO error in handle_watch_update: {str(e)}")
 
 
-def _suppress_werkzeug_ws_abrupt_disconnect_noise():
-    """Patch BaseWSGIServer.log to suppress the AssertionError traceback that fires when
-    a browser closes a WebSocket connection mid-handshake (e.g. closing a tab).
-    The exception is caught inside run_wsgi and routed to self.server.log() — it never
-    propagates out, so wrapping run_wsgi doesn't help. Patching the log method is the
-    only reliable intercept point. The error is cosmetic: Socket.IO already handles the
-    disconnect correctly via its own disconnect handler and timeout logic."""
-    try:
-        from werkzeug.serving import BaseWSGIServer
-        _original_log = BaseWSGIServer.log
-
-        def _filtered_log(self, type, message, *args):
-            if type == 'error' and 'write() before start_response' in message:
-                return
-            _original_log(self, type, message, *args)
-
-        BaseWSGIServer.log = _filtered_log
-    except Exception:
-        pass
-
-
 def init_socketio(app, datastore):
     """Initialize SocketIO with the main Flask app"""
-    _suppress_werkzeug_ws_abrupt_disconnect_noise()
-
     import platform
     import sys
 

changedetectionio/static/js/realtime.js

@@ -116,14 +116,6 @@ $(document).ready(function () {
                 $('#realtime-conn-error').show();
             });
 
-            // Tell the server we're leaving cleanly so it can release the connection
-            // immediately rather than waiting for a timeout.
-            // Note: this only fires for voluntary closes (tab/window close, navigation away).
-            // Hard kills, crashes and network drops will still timeout normally on the server.
-            window.addEventListener('beforeunload', function () {
-                socket.disconnect();
-            });
-
             socket.on('queue_size', function (data) {
                 console.log(`${data.event_timestamp} - Queue size update: ${data.q_length}`);
                 if(queueSizePagerInfoText) {

changedetectionio/tests/test_api.py

@@ -170,14 +170,6 @@ def test_api_simple(client, live_server, measure_memory_usage, datastore_path):
         headers={'x-api-key': api_key},
     )
     assert b'(changed) Which is across' in res.data
-    assert b'Some text thats the same' in res.data
-
-    # Fetch the difference between two versions (default text format)
-    res = client.get(
-        url_for("watchhistorydiff", uuid=watch_uuid, from_timestamp='previous', to_timestamp='latest')+"?changesOnly=true",
-        headers={'x-api-key': api_key},
-    )
-    assert b'Some text thats the same' not in res.data
 
     # Test htmlcolor format
     res = client.get(

changedetectionio/tests/test_group.py

@@ -171,6 +171,7 @@ def test_group_tag_notification(client, live_server, measure_memory_usage, datas
     delete_all_watches(client)
 
     set_original_response(datastore_path=datastore_path)
+    notification_url_endpoint = url_for('test_notification_endpoint', _external=True).replace('http', 'post')
 
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -181,35 +182,50 @@ def test_group_tag_notification(client, live_server, measure_memory_usage, datas
 
     assert b"Watch added" in res.data
 
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
-    notification_form_data = {"notification_urls": notification_url,
-                              "notification_title": "New GROUP TAG ChangeDetection.io Notification - {{watch_url}}",
-                              "notification_body": "BASE URL: {{base_url}}\n"
-                                                   "Watch URL: {{watch_url}}\n"
-                                                   "Watch UUID: {{watch_uuid}}\n"
-                                                   "Watch title: {{watch_title}}\n"
-                                                   "Watch tag: {{watch_tag}}\n"
-                                                   "Preview: {{preview_url}}\n"
-                                                   "Diff URL: {{diff_url}}\n"
-                                                   "Snapshot: {{current_snapshot}}\n"
-                                                   "Diff: {{diff}}\n"
-                                                   "Diff Added: {{diff_added}}\n"
-                                                   "Diff Removed: {{diff_removed}}\n"
-                                                   "Diff Full: {{diff_full}}\n"
-                                                   "Diff as Patch: {{diff_patch}}\n"
-                                                   ":-)",
-                              "notification_screenshot": True,
-                              "notification_format": 'text',
-                              "title": "test-tag"}
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
 
+    group_tag_form_data = {
+        "notification_title": "New GROUP TAG ChangeDetection.io Notification - {{watch_url}}",
+        "notification_body": "BASE URL: {{base_url}}\n"
+                             "Watch URL: {{watch_url}}\n"
+                             "Watch UUID: {{watch_uuid}}\n"
+                             "Watch title: {{watch_title}}\n"
+                             "Watch tag: {{watch_tag}}\n"
+                             "Preview: {{preview_url}}\n"
+                             "Diff URL: {{diff_url}}\n"
+                             "Snapshot: {{current_snapshot}}\n"
+                             "Diff: {{diff}}\n"
+                             "Diff Added: {{diff_added}}\n"
+                             "Diff Removed: {{diff_removed}}\n"
+                             "Diff Full: {{diff_full}}\n"
+                             "Diff as Patch: {{diff_patch}}\n"
+                             ":-)",
+        "notification_screenshot": True,
+        "notification_format": 'text',
+        }
+
+    # Setup for test-tag
+    group_tag_form_data['notification_urls'] = notification_url_endpoint+"?outputfilename=test-tag.txt"
+    group_tag_form_data['title'] = 'test-tag'
     res = client.post(
         url_for("tags.form_tag_edit_submit", uuid=get_UUID_for_tag_name(client, name="test-tag")),
-        data=notification_form_data,
+        data=group_tag_form_data,
+        follow_redirects=True
+    )
+    assert b"Updated" in res.data
+
+    # Setup for other-tag, we only add notifications-urls
+    group_tag_form_data['notification_urls'] = notification_url_endpoint+"?outputfilename=other-tag.txt"
+    group_tag_form_data['title'] = 'other-tag'
+
+    res = client.post(
+        url_for("tags.form_tag_edit_submit", uuid=get_UUID_for_tag_name(client, name="other-tag")),
+        data=group_tag_form_data,
         follow_redirects=True
     )
     assert b"Updated" in res.data
 
-    wait_for_all_checks(client)
 
     set_modified_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -217,12 +233,14 @@ def test_group_tag_notification(client, live_server, measure_memory_usage, datas
 
     time.sleep(3)
 
-    assert os.path.isfile(os.path.join(datastore_path, "notification.txt"))
+    assert os.path.isfile(os.path.join(datastore_path, "test-tag.txt"))
+    assert os.path.isfile(os.path.join(datastore_path, "other-tag.txt"))
 
+    # @todo assert the group name or other unique body is in other-tag.txt
     # Verify what was sent as a notification, this file should exist
-    with open(os.path.join(datastore_path, "notification.txt"), "r") as f:
+    with open(os.path.join(datastore_path, "test-tag.txt"), "r") as f:
         notification_submission = f.read()
-    os.unlink(os.path.join(datastore_path, "notification.txt"))
+    os.unlink(os.path.join(datastore_path, "test-tag.txt"))
 
     # Did we see the URL that had a change, in the notification?
     # Diff was correctly executed

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -16,51 +16,6 @@ except ModuleNotFoundError:
 
 
 
-def test_jsonp_treated_as_plaintext():
-    from ..processors.magic import guess_stream_type
-
-    # JSONP content (server wrongly claims application/json) should be detected as plaintext
-    # Callback names are arbitrary identifiers, not always 'cb'
-    jsonp_content = 'jQuery123456({ "version": "8.0.41", "url": "https://example.com/app.apk" })'
-    result = guess_stream_type(http_content_header="application/json", content=jsonp_content)
-    assert result.is_json is False
-    assert result.is_plaintext is True
-
-    # Variation with dotted callback name e.g. jQuery.cb(...)
-    jsonp_dotted = 'some.callback({ "version": "1.0" })'
-    result = guess_stream_type(http_content_header="application/json", content=jsonp_dotted)
-    assert result.is_json is False
-    assert result.is_plaintext is True
-
-    # Real JSON should still be detected as JSON
-    json_content = '{ "version": "8.0.41", "url": "https://example.com/app.apk" }'
-    result = guess_stream_type(http_content_header="application/json", content=json_content)
-    assert result.is_json is True
-    assert result.is_plaintext is False
-
-
-def test_jsonp_json_filter_extraction():
-    from .. import html_tools
-
-    # Tough case: dotted namespace callback, trailing semicolon, deeply nested content with arrays
-    jsonp_content = 'weixin.update.callback({"platforms": {"android": {"variants": [{"arch": "arm64", "versionName": "8.0.68", "url": "https://example.com/app-arm64.apk"}, {"arch": "arm32", "versionName": "8.0.41", "url": "https://example.com/app-arm32.apk"}]}}});'
-
-    # Deep nested jsonpath filter into array element
-    text = html_tools.extract_json_as_string(jsonp_content, "json:$.platforms.android.variants[0].versionName")
-    assert text == '"8.0.68"'
-
-    # Filter that selects the second array element
-    text = html_tools.extract_json_as_string(jsonp_content, "json:$.platforms.android.variants[1].arch")
-    assert text == '"arm32"'
-
-    if jq_support:
-        text = html_tools.extract_json_as_string(jsonp_content, "jq:.platforms.android.variants[0].versionName")
-        assert text == '"8.0.68"'
-
-        text = html_tools.extract_json_as_string(jsonp_content, "jqraw:.platforms.android.variants[1].url")
-        assert text == "https://example.com/app-arm32.apk"
-
-
 def test_unittest_inline_html_extract():
     # So lets pretend that the JSON we want is inside some HTML
     content="""

changedetectionio/tests/util.py

@@ -343,8 +343,11 @@ def new_live_server_setup(live_server):
     @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET'])
     def test_notification_endpoint():
         datastore_path = current_app.config.get('TEST_DATASTORE_PATH', 'test-datastore')
-
-        with open(os.path.join(datastore_path, "notification.txt"), "wb") as f:
+        from loguru import logger
+        # @todo make safe
+        fname = request.args.get('outputfilename', "notification.txt")
+        logger.debug(f"Writing test notification endpoint data to '{fname}' - {request.args}")
+        with open(os.path.join(datastore_path, fname), "wb") as f:
             # Debug method, dump all POST to file also, used to prove #65
             data = request.stream.read()
             if data != None:

requirements.txt

@@ -40,7 +40,7 @@ orjson~=3.11
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise==1.9.8
+apprise==1.9.7
 
 diff_match_patch