Testing - fix: Replace time.sleep with wait_for_notification_endpoint_output in test_notification (#3716)

README.md

@@ -183,6 +183,9 @@ docker compose pull && docker compose up -d
 
 See the wiki for more information https://github.com/dgtlmoon/changedetection.io/wiki
 
+## Different browser viewport sizes (mobile, desktop etc)
+
+If you are using the recommended `sockpuppetbrowser` (which is in the docker-compose.yml as a setting to be uncommented) you can easily set different viewport sizes for your web page change detection, [see more information here about setting up different viewport sizes](https://github.com/dgtlmoon/sockpuppetbrowser?tab=readme-ov-file#setting-viewport-size).
 
 ## Filters
 

changedetectionio/api/Watch.py

@@ -61,8 +61,17 @@ class Watch(Resource):
     @validate_openapi_request('getWatch')
     def get(self, uuid):
         """Get information about a single watch, recheck, pause, or mute."""
+        import time
         from copy import deepcopy
-        watch = deepcopy(self.datastore.data['watching'].get(uuid))
+        watch = None
+        for _ in range(20):
+            try:
+                watch = deepcopy(self.datastore.data['watching'].get(uuid))
+                break
+            except RuntimeError:
+                # Incase dict changed, try again
+                time.sleep(0.01)
+
         if not watch:
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
 

changedetectionio/blueprint/settings/templates/settings.html

@@ -25,7 +25,7 @@
             <li class="tab"><a href="#ui-options">UI Options</a></li>
             <li class="tab"><a href="#api">API</a></li>
             <li class="tab"><a href="#rss">RSS</a></li>
-            <li class="tab"><a href="#timedate">Time &amp Date</a></li>
+            <li class="tab"><a href="#timedate">Time &amp; Date</a></li>
             <li class="tab"><a href="#proxies">CAPTCHA &amp; Proxies</a></li>
         </ul>
     </div>
@@ -78,9 +78,7 @@
 
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
-                    <div class="field-group">
-                        {{ render_common_settings_form(form.application.form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
-                    </div>
+                    {{ render_common_settings_form(form.application.form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
                 </fieldset>
                 <div class="pure-control-group" id="notification-base-url">
                     {{ render_field(form.application.form.base_url, class="m-d") }}
@@ -121,7 +119,7 @@
                 </div>
                 <div class="pure-control-group">
                     {{ render_field(form.requests.form.timeout) }}
-                    <span class="pure-form-message-inline">For regular plain requests (not chrome based), maximum number of seconds until timeout, 1-999.<br>
+                    <span class="pure-form-message-inline">For regular plain requests (not chrome based), maximum number of seconds until timeout, 1-999.</span><br>
                 </div>
                 <div class="pure-control-group inline-radio">
                     {{ render_field(form.requests.form.default_ua) }}
@@ -212,7 +210,7 @@ nav
                         <a id="chrome-extension-link"
                            title="Try our new Chrome Extension!"
                            href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                            <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='google-chrome-icon.png') }}" alt="Chrome">
+                            <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='google-chrome-icon.png') }}" >
                             Chrome Webstore
                         </a>
                     </p>
@@ -253,14 +251,14 @@ nav
                     Ensure the settings below are correct, they are used to manage the time schedule for checking your web page watches.
                 </div>
                 <div class="pure-control-group">
-                    <p><strong>UTC Time &amp Date from Server:</strong> <span id="utc-time" >{{ utc_time }}</span></p>
-                    <p><strong>Local Time &amp Date in Browser:</strong> <span class="local-time" data-utc="{{ utc_time }}"></span></p>
-                    <p>
+                    <p><strong>UTC Time &amp; Date from Server:</strong> <span id="utc-time" >{{ utc_time }}</span></p>
+                    <p><strong>Local Time &amp; Date in Browser:</strong> <span class="local-time" data-utc="{{ utc_time }}"></span></p>
+                    <div>
                        {{ render_field(form.application.form.scheduler_timezone_default) }}
                         <datalist id="timezones" style="display: none;">
                             {%- for timezone in available_timezones -%}<option value="{{ timezone }}">{{ timezone }}</option>{%- endfor -%}
                         </datalist>
-                    </p>
+                    </div>
                 </div>
             </div>
             <div class="tab-pane-inner" id="ui-options">
@@ -329,7 +327,7 @@ nav
                     </div>
                 </div>
 
-               <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
+                <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.</p>
 
                 <div class="pure-control-group" id="extra-proxies-setting">
                 {{ render_fieldlist_with_inline_errors(form.requests.form.extra_proxies) }}

changedetectionio/blueprint/watchlist/templates/watch-overview.html

@@ -155,7 +155,7 @@ document.addEventListener('DOMContentLoaded', function() {
                     <div class="flex-wrapper">
                     {% if 'favicons_enabled' not in ui_settings or ui_settings['favicons_enabled'] %}
                         <div>{# A page might have hundreds of these images, set IMG options for lazy loading, don't set SRC if we dont have it so it doesnt fetch the placeholder'  #}
-                            <img alt="Favicon thumbnail" class="favicon" loading="lazy" decoding="async" fetchpriority="low" {% if favicon %} src="{{url_for('static_content', group='favicon', filename=watch.uuid)}}" {% else %} src='data:image/svg+xml;utf8,%3Csvg xmlns="http://www.w3.org/2000/svg" width="7.087" height="7.087" viewBox="0 0 7.087 7.087"%3E%3Ccircle cx="3.543" cy="3.543" r="3.279" stroke="%23e1e1e1" stroke-width="0.45" fill="none" opacity="0.74"/%3E%3C/svg%3E' {%  endif %} />
+                            <img alt="Favicon thumbnail" class="favicon" loading="lazy" decoding="async" fetchpriority="low" {% if favicon %} src="{{url_for('static_content', group='favicon', filename=watch.uuid)}}" {% else %} src='data:image/svg+xml;utf8,%3Csvg xmlns="http://www.w3.org/2000/svg" width="7.087" height="7.087" viewBox="0 0 7.087 7.087"%3E%3Ccircle cx="3.543" cy="3.543" r="3.279" stroke="%23e1e1e1" stroke-width="0.45" fill="none" opacity="0.74"/%3E%3C/svg%3E' {%  endif %} >
                         </div>
                     {%  endif %}
                         <div>

changedetectionio/flask_app.py

@@ -680,7 +680,7 @@ def notification_runner():
                 # At the moment only one thread runs (single runner)
                 n_object = notification_q.get(block=False)
             except queue.Empty:
-                time.sleep(1)
+                app.config.exit.wait(1)
 
             else:
 
@@ -717,7 +717,7 @@ def notification_runner():
                         app.config['watch_check_update_SIGNAL'].send(app_context=app, watch_uuid=n_object.get('uuid'))
 
                 # Process notifications
-                notification_debug_log+= ["{} - SENDING - {}".format(now.strftime("%Y/%m/%d %H:%M:%S,000"), json.dumps(sent_obj))]
+                notification_debug_log+= ["{} - SENDING - {}".format(now.strftime("%c"), json.dumps(sent_obj))]
                 # Trim the log length
                 notification_debug_log = notification_debug_log[-100:]
 
@@ -775,7 +775,7 @@ def ticker_thread_check_time_launch_checks():
         # Re #438 - Don't place more watches in the queue to be checked if the queue is already large
         while update_q.qsize() >= 2000:
             logger.warning(f"Recheck watches queue size limit reached ({MAX_QUEUE_SIZE}), skipping adding more items")
-            time.sleep(3)
+            app.config.exit.wait(10.0)
 
 
         recheck_time_system_seconds = int(datastore.threshold_seconds)
@@ -873,8 +873,5 @@ def ticker_thread_check_time_launch_checks():
                     # Reset for next time
                     watch.jitter_seconds = 0
 
-        # Wait before checking the list again - saves CPU
-        time.sleep(1)
-
         # Should be low so we can break this out in testing
         app.config.exit.wait(1)

changedetectionio/templates/_common_fields.html

@@ -145,6 +145,7 @@
                                 <div id="notification-test-log" style="display: none;"><span class="pure-form-message-inline">Processing..</span></div>
                             </div>
                         </div>
+
                         <div class="pure-control-group grey-form-border">
                             <div class="pure-control-group">
                                 {{ render_field(form.notification_title, class="m-d notification-title", placeholder=settings_application['notification_title']) }}
@@ -169,6 +170,7 @@
                                     </span></li>
                                     </ul>
                                     <br>
+                                </div>
                             </div>
                             <div class="">
                                 {{ render_field(form.notification_format , class="notification-format") }}

changedetectionio/tests/test_notification.py

@@ -5,7 +5,7 @@ import re
 from flask import url_for
 from loguru import logger
 
-from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks
+from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -83,7 +83,9 @@ def test_check_notification(client, live_server, measure_memory_usage, datastore
 
 
     uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-    with open(os.path.join(datastore_path, str(uuid), 'last-screenshot.png'), 'wb') as f:
+    screenshot_dir = os.path.join(datastore_path, str(uuid))
+    os.makedirs(screenshot_dir, exist_ok=True)
+    with open(os.path.join(screenshot_dir, 'last-screenshot.png'), 'wb') as f:
         f.write(base64.b64decode(testimage_png))
 
     # Goto the edit page, add our ignore text
@@ -142,7 +144,7 @@ def test_check_notification(client, live_server, measure_memory_usage, datastore
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    time.sleep(3)
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
 
     # Check no errors were recorded
     res = client.get(url_for("watchlist.index"))
@@ -198,7 +200,8 @@ def test_check_notification(client, live_server, measure_memory_usage, datastore
     # This should insert the {current_snapshot}
     set_more_modified_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    time.sleep(3)
+    wait_for_all_checks(client)
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
     # Verify what was sent as a notification, this file should exist
     with open(os.path.join(datastore_path, "notification.txt"), "r") as f:
         notification_submission = f.read()
@@ -239,7 +242,8 @@ def test_check_notification(client, live_server, measure_memory_usage, datastore
     )
     assert b"Updated watch." in res.data
 
-    time.sleep(2)
+    wait_for_all_checks(client)
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
 
     # Verify what was sent as a notification, this file should exist
     with open(os.path.join(datastore_path, "notification.txt"), "r") as f:
@@ -324,7 +328,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
-    time.sleep(2) # plus extra delay for notifications to fire
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
 
 
     # Check no errors were recorded, because we asked for 204 which is slightly uncommon but is still OK
@@ -442,7 +446,7 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert res.status_code != 500
 
     # Give apprise time to fire
-    time.sleep(4)
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
 
     with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
         x = f.read()
@@ -571,7 +575,7 @@ def _test_color_notifications(client, notification_body_token, datastore_path):
     assert b'Queued 1 watch for rechecking.' in res.data
 
     wait_for_all_checks(client)
-    time.sleep(3)
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
 
     with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
         x = f.read()