also dump memory report when building

.github/workflows/pypi-release.yml

@@ -28,7 +28,7 @@ jobs:
 
 
   test-pypi-package:
-    name: Test the built package works basically.
+    name: Test the built 📦 package works basically.
     runs-on: ubuntu-latest
     needs:
     - build
@@ -42,39 +42,18 @@ jobs:
       uses: actions/setup-python@v6
       with:
         python-version: '3.11'
-
     - name: Test that the basic pip built package runs without error
       run: |
         set -ex
         ls -alR 
         
-        # Install the first wheel found in dist/
-        WHEEL=$(find dist -type f -name "*.whl" -print -quit)
-        echo Installing $WHEEL
-        python3 -m pip install --upgrade pip
-        python3 -m pip install "$WHEEL"
+        # Find and install the first .whl file
+        find dist -type f -name "*.whl" -exec pip3 install {} \; -quit
         changedetection.io -d /tmp -p 10000 &
         
         sleep 3
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/ >/dev/null
-        
-        # --- API test ---
-        # This also means that the docs/api-spec.yml was shipped and could be read
-        test -f /tmp/url-watches.json
-        API_KEY=$(jq -r '.. | .api_access_token? // empty' /tmp/url-watches.json)
-        echo Test API KEY is $API_KEY
-        curl -X POST "http://127.0.0.1:10000/api/v1/watch" \
-          -H "x-api-key: ${API_KEY}" \
-          -H "Content-Type: application/json" \
-          --show-error --fail \
-          --retry 6 --retry-delay 1 --retry-connrefused \
-          -d '{
-            "url": "https://example.com",
-            "title": "Example Site Monitor",
-            "time_between_check": { "hours": 1 }
-          }'
-          
         killall changedetection.io
 
 

.github/workflows/test-stack-reusable-workflow.yml

@@ -54,10 +54,7 @@ jobs:
 
       - name: Spin up ancillary SMTP+Echo message test server
         run: |
-          # Debug SMTP server/echo message back server, telnet 11080 to it should immediately bounce back the most recent message that tried to send (then you can see if cdio tried to send, the format, etc)
-          # 11025 is the SMTP port for testing
-          # apprise example would be 'mailto://changedetection@localhost:11025/?to=fff@home.com  (it will also echo to STDOUT)
-          # telnet localhost 11080
+          # Debug SMTP server/echo message back server
           docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'pip3 install aiosmtpd && python changedetectionio/tests/smtp/smtp-test-server.py'
           docker ps
 
@@ -249,36 +246,20 @@ jobs:
           # @todo - scan the container log to see the right "graceful shutdown" text exists           
           docker rm sig-test
 
-      - name: Dump container log
+      - name: Dump container log and memory report
         if: always()
         run: |
           mkdir output-logs
           docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout-${{ env.PYTHON_VERSION }}.txt
           docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr-${{ env.PYTHON_VERSION }}.txt
 
-      - name: Extract and display memory test report
-        if: always()
-        run: |
           # Extract test-memory.log from the container
           echo "Extracting test-memory.log from container..."
           docker cp test-cdio-basic-tests:/app/changedetectionio/test-memory.log output-logs/test-memory-${{ env.PYTHON_VERSION }}.log || echo "test-memory.log not found in container"
 
           # Display the memory log contents for immediate visibility in workflow output
-          echo "=== Top 10 Highest Peak Memory Tests ==="
-          if [ -f output-logs/test-memory-${{ env.PYTHON_VERSION }}.log ]; then
-            # Sort by peak memory value (extract number before MB and sort numerically, reverse order)
-            grep "Peak memory:" output-logs/test-memory-${{ env.PYTHON_VERSION }}.log | \
-              sed 's/.*Peak memory: //' | \
-              paste -d'|' - <(grep "Peak memory:" output-logs/test-memory-${{ env.PYTHON_VERSION }}.log) | \
-              sort -t'|' -k1 -nr | \
-              cut -d'|' -f2 | \
-              head -10
-            echo ""
-            echo "=== Full Memory Test Report ==="
-            cat output-logs/test-memory-${{ env.PYTHON_VERSION }}.log
-          else
-            echo "No memory log available"
-          fi
+          echo "=== Memory Test Report ==="
+          cat output-logs/test-memory-${{ env.PYTHON_VERSION }}.log 2>/dev/null || echo "No memory log available"
 
       - name: Store everything including test-datastore
         if: always()

MANIFEST.in

@@ -1,9 +1,7 @@
 recursive-include changedetectionio/api *
-include docs/api-spec.yaml
 recursive-include changedetectionio/blueprint *
 recursive-include changedetectionio/conditions *
 recursive-include changedetectionio/content_fetchers *
-recursive-include changedetectionio/jinja2_custom *
 recursive-include changedetectionio/model *
 recursive-include changedetectionio/notification *
 recursive-include changedetectionio/processors *

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.50.27'
+__version__ = '0.50.21'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/api/__init__.py

@@ -37,10 +37,6 @@ def get_openapi_spec():
     from openapi_core import OpenAPI  # Lazy import - saves ~10.7 MB on startup
 
     spec_path = os.path.join(os.path.dirname(__file__), '../../docs/api-spec.yaml')
-    if not os.path.exists(spec_path):
-        # Possibly for pip3 packages
-        spec_path = os.path.join(os.path.dirname(__file__), '../docs/api-spec.yaml')
-
     with open(spec_path, 'r') as f:
         spec_dict = yaml.safe_load(f)
     _openapi_spec = OpenAPI.from_dict(spec_dict)

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -6,7 +6,7 @@ from loguru import logger
 
 from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT
 from changedetectionio.content_fetchers.base import manage_user_agent
-from changedetectionio.jinja2_custom import render as jinja_render
+from changedetectionio.safe_jinja import render as jinja_render
 
 
 

changedetectionio/blueprint/check_proxies/__init__.py

@@ -33,7 +33,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def long_task(uuid, preferred_proxy):
         import time
         from changedetectionio.content_fetchers import exceptions as content_fetcher_exceptions
-        from changedetectionio.jinja2_custom import render as jinja_render
+        from changedetectionio.safe_jinja import render as jinja_render
 
         status = {'status': '', 'length': 0, 'text': ''}
 

changedetectionio/blueprint/rss/blueprint.py

@@ -1,5 +1,5 @@
 
-from changedetectionio.jinja2_custom import render as jinja_render
+from changedetectionio.safe_jinja import render as jinja_render
 from changedetectionio.store import ChangeDetectionStore
 from feedgen.feed import FeedGenerator
 from flask import Blueprint, make_response, request, url_for, redirect

changedetectionio/blueprint/settings/__init__.py

@@ -119,7 +119,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                                 hide_remove_pass=os.getenv("SALTED_PASS", False),
                                 min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)),
                                 settings_application=datastore.data['settings']['application'],
-                                timezone_default_config=datastore.data['settings']['application'].get('scheduler_timezone_default'),
+                                timezone_default_config=datastore.data['settings']['application'].get('timezone'),
                                 utc_time=utc_time,
                                 )
 

changedetectionio/blueprint/settings/templates/settings.html

@@ -1,7 +1,7 @@
 {% extends 'base.html' %}
 
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, render_ternary_field, render_fieldlist_with_inline_errors %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, render_ternary_field %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script>
     const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="global-settings")}}";
@@ -89,6 +89,15 @@
                             <span class="pure-form-message-inline">Transforms RSS/RDF feed watches into beautiful text only</span>
                         </div>
                     </div>
+
+                {% if form.requests.proxy %}
+                    <div class="pure-control-group inline-radio">
+                        {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
+                        <span class="pure-form-message-inline">
+                        Choose a default proxy for all watches
+                        </span>
+                    </div>
+                {% endif %}
                 </fieldset>
             </div>
 
@@ -131,10 +140,6 @@
                     <span class="pure-form-message-inline">Number of concurrent workers to process watches. More workers = faster processing but higher memory usage.<br>
                     Currently running: <strong>{{ worker_info.count }}</strong> operational {{ worker_info.type }} workers{% if worker_info.active_workers > 0 %} ({{ worker_info.active_workers }} actively processing){% endif %}.</span>
                 </div>
-                <div class="pure-control-group">
-                    {{ render_field(form.requests.form.timeout) }}
-                    <span class="pure-form-message-inline">For regular plain requests (not chrome based), maximum number of seconds until timeout, 1-999.<br>
-                </div>
                 <div class="pure-control-group inline-radio">
                     {{ render_field(form.requests.form.default_ua) }}
                     <span class="pure-form-message-inline">
@@ -238,7 +243,7 @@ nav
                     <p><strong>UTC Time &amp Date from Server:</strong> <span id="utc-time" >{{ utc_time }}</span></p>
                     <p><strong>Local Time &amp Date in Browser:</strong> <span class="local-time" data-utc="{{ utc_time }}"></span></p>
                     <p>
-                       {{ render_field(form.application.form.scheduler_timezone_default) }}
+                       {{ render_field(form.application.form.timezone) }}
                         <datalist id="timezones" style="display: none;">
                             {% for tz_name in available_timezones %}
                                 <option value="{{ tz_name }}">{{ tz_name }}</option>
@@ -316,27 +321,17 @@ nav
                <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
 
                 <div class="pure-control-group" id="extra-proxies-setting">
-                {{ render_fieldlist_with_inline_errors(form.requests.form.extra_proxies) }}
+                {{ render_field(form.requests.form.extra_proxies) }}
                 <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span><br>
                 <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
-                {% if form.requests.proxy %}
-                <div>
-                <br>
-                    <div class="inline-radio">
-                        {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
-                        <span class="pure-form-message-inline">Choose a default proxy for all watches</span>
-                    </div>
-                </div>
-                {% endif %}
                 </div>
                 <div class="pure-control-group" id="extra-browsers-setting">
                     <p>
                     <span class="pure-form-message-inline"><i>Extra Browsers</i> can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.</span><br>
                     <span class="pure-form-message-inline">Simply paste the connection address into the box, <a href="https://changedetection.io/tutorial/using-bright-datas-scraping-browser-pass-captchas-and-other-protection-when-monitoring">More instructions and examples here</a> </span>
                     </p>
-                    {{ render_fieldlist_with_inline_errors(form.requests.form.extra_browsers) }}
+                    {{ render_field(form.requests.form.extra_browsers) }}
                 </div>
-            
             </div>
             <div id="actions">
                 <div class="pure-control-group">

changedetectionio/blueprint/ui/edit.py

@@ -187,7 +187,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
             tz_name = time_schedule_limit.get('timezone')
             if not tz_name:
-                tz_name = datastore.data['settings']['application'].get('scheduler_timezone_default', os.getenv('TZ', 'UTC').strip())
+                tz_name = datastore.data['settings']['application'].get('timezone', 'UTC')
 
             if time_schedule_limit and time_schedule_limit.get('enabled'):
                 try:
@@ -257,7 +257,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                 'system_has_webdriver_configured': os.getenv('WEBDRIVER_URL'),
                 'ui_edit_stats_extras': collect_ui_edit_stats_extras(watch),
                 'visual_selector_data_ready': datastore.visualselector_data_is_ready(watch_uuid=uuid),
-                'timezone_default_config': datastore.data['settings']['application'].get('scheduler_timezone_default'),
+                'timezone_default_config': datastore.data['settings']['application'].get('timezone'),
                 'using_global_webdriver_wait': not default['webdriver_delay'],
                 'uuid': uuid,
                 'watch': watch,

changedetectionio/blueprint/ui/notification.py

@@ -2,7 +2,6 @@ from flask import Blueprint, request, make_response
 import random
 from loguru import logger
 
-from changedetectionio.notification_service import NotificationContextData
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.auth_decorator import login_optionally_required
 
@@ -20,7 +19,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         import apprise
         from changedetectionio.notification.handler import process_notification
         from changedetectionio.notification.apprise_plugin.assets import apprise_asset
-        from changedetectionio.jinja2_custom import render as jinja_render
 
         from changedetectionio.notification.apprise_plugin.custom_handlers import apprise_http_custom_handler
 
@@ -63,20 +61,16 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             return 'Error: No Notification URLs set/found'
 
         for n_url in notification_urls:
-            # We are ONLY validating the apprise:// part here, convert all tags to something so as not to break apprise URLs
-            generic_notification_context_data = NotificationContextData()
-            generic_notification_context_data.set_random_for_validation()
-            n_url = jinja_render(template_str=n_url, **generic_notification_context_data).strip()
             if len(n_url.strip()):
                 if not apobj.add(n_url):
                     return f'Error:  {n_url} is not a valid AppRise URL.'
 
         try:
             # use the same as when it is triggered, but then override it with the form test values
-            n_object = NotificationContextData({
+            n_object = {
                 'watch_url': request.form.get('window_url', "https://changedetection.io"),
                 'notification_urls': notification_urls
-            })
+            }
 
             # Only use if present, if not set in n_object it should use the default system value
             if 'notification_format' in request.form and request.form['notification_format'].strip():

changedetectionio/content_fetchers/base.py

@@ -141,7 +141,7 @@ class Fetcher():
     async def iterate_browser_steps(self, start_url=None):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
         from playwright._impl._errors import TimeoutError, Error
-        from changedetectionio.jinja2_custom import render as jinja_render
+        from changedetectionio.safe_jinja import render as jinja_render
         step_n = 0
 
         if self.browser_steps is not None and len(self.browser_steps):

changedetectionio/content_fetchers/requests.py

@@ -51,7 +51,6 @@ class fetcher(Fetcher):
 
         session = requests.Session()
 
-
         if strtobool(os.getenv('ALLOW_FILE_URI', 'false')) and url.startswith('file://'):
             from requests_file import FileAdapter
             session.mount('file://', FileAdapter())

changedetectionio/flask_app.py

@@ -795,7 +795,7 @@ def ticker_thread_check_time_launch_checks():
             else:
                 time_schedule_limit = watch.get('time_schedule_limit')
                 logger.trace(f"{uuid} Time scheduler - Using watch settings (not global settings)")
-            tz_name = datastore.data['settings']['application'].get('scheduler_timezone_default', os.getenv('TZ', 'UTC').strip())
+            tz_name = datastore.data['settings']['application'].get('timezone', 'UTC')
 
             if time_schedule_limit and time_schedule_limit.get('enabled'):
                 try:

changedetectionio/forms.py

@@ -5,7 +5,6 @@ from wtforms.widgets.core import TimeInput
 
 from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES
 from changedetectionio.conditions.form import ConditionFormRow
-from changedetectionio.notification_service import NotificationContextData
 from changedetectionio.strtobool import strtobool
 
 from wtforms import (
@@ -470,16 +469,11 @@ class ValidateAppRiseServers(object):
         import apprise
         from .notification.apprise_plugin.assets import apprise_asset
         from .notification.apprise_plugin.custom_handlers import apprise_http_custom_handler  # noqa: F401
-        from changedetectionio.jinja2_custom import render as jinja_render
 
         apobj = apprise.Apprise(asset=apprise_asset)
 
         for server_url in field.data:
-            generic_notification_context_data = NotificationContextData()
-            # Make sure something is atleast in all those regular token fields
-            generic_notification_context_data.set_random_for_validation()
-
-            url = jinja_render(template_str=server_url.strip(), **generic_notification_context_data).strip()
+            url = server_url.strip()
             if url.startswith("#"):
                 continue
 
@@ -493,8 +487,9 @@ class ValidateJinja2Template(object):
     """
     def __call__(self, form, field):
         from changedetectionio import notification
-        from changedetectionio.jinja2_custom import create_jinja_env
+
         from jinja2 import BaseLoader, TemplateSyntaxError, UndefinedError
+        from jinja2.sandbox import ImmutableSandboxedEnvironment
         from jinja2.meta import find_undeclared_variables
         import jinja2.exceptions
 
@@ -502,11 +497,9 @@ class ValidateJinja2Template(object):
         joined_data = ' '.join(map(str, field.data)) if isinstance(field.data, list) else f"{field.data}"
 
         try:
-            # Use the shared helper to create a properly configured environment
-            jinja2_env = create_jinja_env(loader=BaseLoader)
-
-            # Add notification tokens for validation
-            jinja2_env.globals.update(NotificationContextData())
+            jinja2_env = ImmutableSandboxedEnvironment(loader=BaseLoader, extensions=['jinja2_time.TimeExtension'])
+            jinja2_env.globals.update(notification.valid_tokens)
+            # Extra validation tokens provided on the form_class(... extra_tokens={}) setup
             if hasattr(field, 'extra_notification_tokens'):
                 jinja2_env.globals.update(field.extra_notification_tokens)
 
@@ -518,7 +511,6 @@ class ValidateJinja2Template(object):
         except jinja2.exceptions.SecurityError as e:
             raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
 
-        # Check for undeclared variables
         ast = jinja2_env.parse(joined_data)
         undefined = ", ".join(find_undeclared_variables(ast))
         if undefined:
@@ -686,51 +678,6 @@ class ValidateCSSJSONXPATHInput(object):
                 except:
                     raise ValidationError("A system-error occurred when validating your jq expression")
 
-class ValidateSimpleURL:
-    """Validate that the value can be parsed by urllib.parse.urlparse() and has a scheme/netloc."""
-    def __init__(self, message=None):
-        self.message = message or "Invalid URL."
-
-    def __call__(self, form, field):
-        data = (field.data or "").strip()
-        if not data:
-            return  # empty is OK — pair with validators.Optional()
-        from urllib.parse import urlparse
-
-        parsed = urlparse(data)
-        if not parsed.scheme or not parsed.netloc:
-            raise ValidationError(self.message)
-
-class ValidateStartsWithRegex(object):
-    def __init__(self, regex, *, flags=0, message=None, allow_empty=True, split_lines=True):
-        # compile with given flags (we’ll pass re.IGNORECASE below)
-        self.pattern = re.compile(regex, flags) if isinstance(regex, str) else regex
-        self.message = message
-        self.allow_empty = allow_empty
-        self.split_lines = split_lines
-
-    def __call__(self, form, field):
-        data = field.data
-        if not data:
-            return
-
-        # normalize into list of lines
-        if isinstance(data, str) and self.split_lines:
-            lines = data.splitlines()
-        elif isinstance(data, (list, tuple)):
-            lines = data
-        else:
-            lines = [data]
-
-        for line in lines:
-            stripped = line.strip()
-            if not stripped:
-                if self.allow_empty:
-                    continue
-                raise ValidationError(self.message or "Empty value not allowed.")
-            if not self.pattern.match(stripped):
-                raise ValidationError(self.message or "Invalid value.")
-
 class quickWatchForm(Form):
     from . import processors
 
@@ -758,7 +705,7 @@ class commonSettingsForm(Form):
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
     processor = RadioField( label=u"Processor - What do you want to achieve?", choices=processors.available_processors(), default="text_json_diff")
-    scheduler_timezone_default = StringField("Default timezone for watch check scheduler", render_kw={"list": "timezones"}, validators=[validateTimeZoneName()])
+    timezone = StringField("Timezone for watch schedule", render_kw={"list": "timezones"}, validators=[validateTimeZoneName()])
     webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")])
 
 
@@ -848,7 +795,7 @@ class processor_text_json_diff_form(commonSettingsForm):
         if not super().validate():
             return False
 
-        from changedetectionio.jinja2_custom import render as jinja_render
+        from changedetectionio.safe_jinja import render as jinja_render
         result = True
 
         # Fail form validation when a body is set for a GET
@@ -911,36 +858,23 @@ class processor_text_json_diff_form(commonSettingsForm):
     ):
         super().__init__(formdata, obj, prefix, data, meta, **kwargs)
         if kwargs and kwargs.get('default_system_settings'):
-            default_tz = kwargs.get('default_system_settings').get('application', {}).get('scheduler_timezone_default')
+            default_tz = kwargs.get('default_system_settings').get('application', {}).get('timezone')
             if default_tz:
                 self.time_schedule_limit.form.timezone.render_kw['placeholder'] = default_tz
 
 
 
 class SingleExtraProxy(Form):
+
     # maybe better to set some <script>var..
     proxy_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
-    proxy_url = StringField('Proxy URL', [
-        validators.Optional(),
-        ValidateStartsWithRegex(
-            regex=r'^(https?|socks5)://',  # ✅ main pattern
-            flags=re.IGNORECASE,  # ✅ makes it case-insensitive
-            message='Proxy URLs must start with http://, https:// or socks5://',
-        ),
-        ValidateSimpleURL()
-    ], render_kw={"placeholder": "socks5:// or regular proxy http://user:pass@...:3128", "size":50})
+    proxy_url = StringField('Proxy URL', [validators.Optional()], render_kw={"placeholder": "socks5:// or regular proxy http://user:pass@...:3128", "size":50})
+    # @todo do the validation here instead
 
 class SingleExtraBrowser(Form):
     browser_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
-    browser_connection_url = StringField('Browser connection URL', [
-        validators.Optional(),
-        ValidateStartsWithRegex(
-            regex=r'^(wss?|ws)://',
-            flags=re.IGNORECASE,
-            message='Browser URLs must start with wss:// or ws://'
-        ),
-        ValidateSimpleURL()
-    ], render_kw={"placeholder": "wss://brightdata... wss://oxylabs etc", "size":50})
+    browser_connection_url = StringField('Browser connection URL', [validators.Optional()], render_kw={"placeholder": "wss://brightdata... wss://oxylabs etc", "size":50})
+    # @todo do the validation here instead
 
 class DefaultUAInputForm(Form):
     html_requests = StringField('Plaintext requests', validators=[validators.Optional()], render_kw={"placeholder": "<default>"})
@@ -951,7 +885,7 @@ class DefaultUAInputForm(Form):
 class globalSettingsRequestForm(Form):
     time_between_check = RequiredFormField(TimeBetweenCheckForm)
     time_schedule_limit = FormField(ScheduleLimitForm)
-    proxy = RadioField('Default proxy')
+    proxy = RadioField('Proxy')
     jitter_seconds = IntegerField('Random jitter seconds ± check',
                                   render_kw={"style": "width: 5em;"},
                                   validators=[validators.NumberRange(min=0, message="Should contain zero or more seconds")])
@@ -960,12 +894,7 @@ class globalSettingsRequestForm(Form):
                           render_kw={"style": "width: 5em;"},
                           validators=[validators.NumberRange(min=1, max=50,
                                                              message="Should be between 1 and 50")])
-
-    timeout = IntegerField('Requests timeout in seconds',
-                           render_kw={"style": "width: 5em;"},
-                           validators=[validators.NumberRange(min=1, max=999,
-                                                              message="Should be between 1 and 999")])
-
+    
     extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5)
     extra_browsers = FieldList(FormField(SingleExtraBrowser), min_entries=5)
 

changedetectionio/html_tools.py

@@ -408,9 +408,6 @@ def strip_ignore_text(content, wordlist, mode="content"):
     ignored_lines = []
 
     for k in wordlist:
-        # Skip empty strings to avoid matching everything
-        if not k or not k.strip():
-            continue
         # Is it a regex?
         res = re.search(PERL_STYLE_REGEX, k, re.IGNORECASE)
         if res:

changedetectionio/jinja2_custom/__init__.py

@@ -1,20 +0,0 @@
-"""
-Jinja2 custom extensions and safe rendering utilities.
-"""
-from .extensions.TimeExtension import TimeExtension
-from .safe_jinja import (
-    render,
-    render_fully_escaped,
-    create_jinja_env,
-    JINJA2_MAX_RETURN_PAYLOAD_SIZE,
-    DEFAULT_JINJA2_EXTENSIONS,
-)
-
-__all__ = [
-    'TimeExtension',
-    'render',
-    'render_fully_escaped',
-    'create_jinja_env',
-    'JINJA2_MAX_RETURN_PAYLOAD_SIZE',
-    'DEFAULT_JINJA2_EXTENSIONS',
-]

changedetectionio/jinja2_custom/extensions/TimeExtension.py

@@ -1,221 +0,0 @@
-"""
-Jinja2 TimeExtension - Custom date/time handling for templates.
-
-This extension provides the {% now %} tag for Jinja2 templates, offering timezone-aware
-date/time formatting with support for time offsets.
-
-Why This Extension Exists:
-    The Arrow library has a now() function (arrow.now()), but Jinja2 templates cannot
-    directly call Python functions - they need extensions or filters to expose functionality.
-
-    This TimeExtension serves as a Jinja2-to-Arrow bridge that:
-
-    1. Makes Arrow accessible in templates - Jinja2 requires registering functions/tags
-       through extensions. You cannot use arrow.now() directly in a template.
-
-    2. Provides template-friendly syntax - Instead of complex Python code, you get clean tags:
-       {% now 'UTC' %}
-       {% now 'UTC' + 'hours=2' %}
-       {% now 'Europe/London', '%Y-%m-%d' %}
-
-    3. Adds convenience features on top of Arrow:
-       - Default timezone from environment variable (TZ) or config
-       - Default datetime format configuration
-       - Offset syntax parsing: 'hours=2,minutes=30' → shift(hours=2, minutes=30)
-       - Empty string timezone support to use configured defaults
-
-    4. Maintains security - Works within Jinja2's sandboxed environment so users
-       cannot access arbitrary Python code or objects.
-
-    Essentially, this is a Jinja2 wrapper around arrow.now() and arrow.shift() that
-    provides user-friendly template syntax while maintaining security.
-
-Basic Usage:
-    {% now 'UTC' %}
-    # Output: Wed, 09 Dec 2015 23:33:01
-
-Custom Format:
-    {% now 'UTC', '%Y-%m-%d %H:%M:%S' %}
-    # Output: 2015-12-09 23:33:01
-
-Timezone Support:
-    {% now 'America/New_York' %}
-    {% now 'Europe/London' %}
-    {% now '' %}  # Uses default timezone from environment.default_timezone
-
-Time Offsets (Addition):
-    {% now 'UTC' + 'hours=2' %}
-    {% now 'UTC' + 'hours=2,minutes=30' %}
-    {% now 'UTC' + 'days=1,hours=2,minutes=15,seconds=10' %}
-
-Time Offsets (Subtraction):
-    {% now 'UTC' - 'minutes=11' %}
-    {% now 'UTC' - 'days=2,minutes=33,seconds=1' %}
-
-Time Offsets with Custom Format:
-    {% now 'UTC' + 'hours=2', '%Y-%m-%d %H:%M:%S' %}
-    # Output: 2015-12-10 01:33:01
-
-Weekday Support (for finding next/previous weekday):
-    {% now 'UTC' + 'weekday=0' %}  # Next Monday (0=Monday, 6=Sunday)
-    {% now 'UTC' + 'weekday=4' %}  # Next Friday
-
-Configuration:
-    - Default timezone: Set via TZ environment variable or override environment.default_timezone
-    - Default format: '%a, %d %b %Y %H:%M:%S' (can be overridden via environment.datetime_format)
-
-Environment Customization:
-    from changedetectionio.jinja2_custom import create_jinja_env
-
-    jinja2_env = create_jinja_env()
-    jinja2_env.default_timezone = 'America/New_York'  # Override default timezone
-    jinja2_env.datetime_format = '%Y-%m-%d %H:%M'      # Override default format
-
-Supported Offset Parameters:
-    - years, months, weeks, days
-    - hours, minutes, seconds, microseconds
-    - weekday (0=Monday through 6=Sunday, must be integer)
-
-Note:
-    This extension uses the Arrow library for timezone-aware datetime handling.
-    All timezone names should be valid IANA timezone identifiers (e.g., 'America/New_York').
-"""
-import arrow
-
-from jinja2 import nodes
-from jinja2.ext import Extension
-import os
-
-class TimeExtension(Extension):
-    """
-    Jinja2 Extension providing the {% now %} tag for timezone-aware date/time rendering.
-
-    This extension adds two attributes to the Jinja2 environment:
-    - datetime_format: Default strftime format string (default: '%a, %d %b %Y %H:%M:%S')
-    - default_timezone: Default timezone for rendering (default: TZ env var or 'UTC')
-
-    Both can be overridden after environment creation by setting the attributes directly.
-    """
-
-    tags = {'now'}
-
-    def __init__(self, environment):
-        """Jinja2 Extension constructor."""
-        super().__init__(environment)
-
-        environment.extend(
-            datetime_format='%a, %d %b %Y %H:%M:%S',
-            default_timezone=os.getenv('TZ', 'UTC').strip()
-        )
-
-    def _datetime(self, timezone, operator, offset, datetime_format):
-        """
-        Get current datetime with time offset applied.
-
-        Args:
-            timezone: IANA timezone identifier (e.g., 'UTC', 'America/New_York') or empty string for default
-            operator: '+' for addition or '-' for subtraction
-            offset: Comma-separated offset parameters (e.g., 'hours=2,minutes=30')
-            datetime_format: strftime format string or None to use environment default
-
-        Returns:
-            Formatted datetime string with offset applied
-
-        Example:
-            _datetime('UTC', '+', 'hours=2,minutes=30', '%Y-%m-%d %H:%M:%S')
-            # Returns current time + 2.5 hours
-        """
-        # Use default timezone if none specified
-        if not timezone or timezone == '':
-            timezone = self.environment.default_timezone
-
-        d = arrow.now(timezone)
-
-        # parse shift params from offset and include operator
-        shift_params = {}
-        for param in offset.split(','):
-            interval, value = param.split('=')
-            shift_params[interval.strip()] = float(operator + value.strip())
-
-        # Fix weekday parameter can not be float
-        if 'weekday' in shift_params:
-            shift_params['weekday'] = int(shift_params['weekday'])
-
-        d = d.shift(**shift_params)
-
-        if datetime_format is None:
-            datetime_format = self.environment.datetime_format
-        return d.strftime(datetime_format)
-
-    def _now(self, timezone, datetime_format):
-        """
-        Get current datetime without any offset.
-
-        Args:
-            timezone: IANA timezone identifier (e.g., 'UTC', 'America/New_York') or empty string for default
-            datetime_format: strftime format string or None to use environment default
-
-        Returns:
-            Formatted datetime string for current time
-
-        Example:
-            _now('America/New_York', '%Y-%m-%d %H:%M:%S')
-            # Returns current time in New York timezone
-        """
-        # Use default timezone if none specified
-        if not timezone or timezone == '':
-            timezone = self.environment.default_timezone
-
-        if datetime_format is None:
-            datetime_format = self.environment.datetime_format
-        return arrow.now(timezone).strftime(datetime_format)
-
-    def parse(self, parser):
-        """
-        Parse the {% now %} tag and generate appropriate AST nodes.
-
-        This method is called by Jinja2 when it encounters a {% now %} tag.
-        It parses the tag syntax and determines whether to call _now() or _datetime()
-        based on whether offset operations (+ or -) are present.
-
-        Supported syntax:
-            {% now 'timezone' %}                              -> calls _now()
-            {% now 'timezone', 'format' %}                    -> calls _now()
-            {% now 'timezone' + 'offset' %}                   -> calls _datetime()
-            {% now 'timezone' + 'offset', 'format' %}         -> calls _datetime()
-            {% now 'timezone' - 'offset', 'format' %}         -> calls _datetime()
-
-        Args:
-            parser: Jinja2 parser instance
-
-        Returns:
-            nodes.Output: AST output node containing the formatted datetime string
-        """
-        lineno = next(parser.stream).lineno
-
-        node = parser.parse_expression()
-
-        if parser.stream.skip_if('comma'):
-            datetime_format = parser.parse_expression()
-        else:
-            datetime_format = nodes.Const(None)
-
-        if isinstance(node, nodes.Add):
-            call_method = self.call_method(
-                '_datetime',
-                [node.left, nodes.Const('+'), node.right, datetime_format],
-                lineno=lineno,
-            )
-        elif isinstance(node, nodes.Sub):
-            call_method = self.call_method(
-                '_datetime',
-                [node.left, nodes.Const('-'), node.right, datetime_format],
-                lineno=lineno,
-            )
-        else:
-            call_method = self.call_method(
-                '_now',
-                [node, datetime_format],
-                lineno=lineno,
-            )
-        return nodes.Output([call_method], lineno=lineno)

changedetectionio/jinja2_custom/safe_jinja.py

@@ -1,55 +0,0 @@
-"""
-Safe Jinja2 render with max payload sizes
-
-See https://jinja.palletsprojects.com/en/3.1.x/sandbox/#security-considerations
-"""
-
-import jinja2.sandbox
-import typing as t
-import os
-from .extensions.TimeExtension import TimeExtension
-
-JINJA2_MAX_RETURN_PAYLOAD_SIZE = 1024 * int(os.getenv("JINJA2_MAX_RETURN_PAYLOAD_SIZE_KB", 1024 * 10))
-
-# Default extensions - can be overridden in create_jinja_env()
-DEFAULT_JINJA2_EXTENSIONS = [TimeExtension]
-
-
-def create_jinja_env(extensions=None, **kwargs) -> jinja2.sandbox.ImmutableSandboxedEnvironment:
-    """
-    Create a sandboxed Jinja2 environment with our custom extensions and default timezone.
-
-    Args:
-        extensions: List of extension classes to use (defaults to DEFAULT_JINJA2_EXTENSIONS)
-        **kwargs: Additional arguments to pass to ImmutableSandboxedEnvironment
-
-    Returns:
-        Configured Jinja2 environment
-    """
-    if extensions is None:
-        extensions = DEFAULT_JINJA2_EXTENSIONS
-
-    jinja2_env = jinja2.sandbox.ImmutableSandboxedEnvironment(
-        extensions=extensions,
-        **kwargs
-    )
-
-    # Get default timezone from environment variable
-    default_timezone = os.getenv('TZ', 'UTC').strip()
-    jinja2_env.default_timezone = default_timezone
-
-    return jinja2_env
-
-
-# This is used for notifications etc, so actually it's OK to send custom HTML such as <a href> etc, but it should limit what data is available.
-# (Which also limits available functions that could be called)
-def render(template_str, **args: t.Any) -> str:
-    jinja2_env = create_jinja_env()
-    output = jinja2_env.from_string(template_str).render(args)
-    return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]
-
-def render_fully_escaped(content):
-    env = jinja2.sandbox.ImmutableSandboxedEnvironment(autoescape=True)
-    template = env.from_string("{{ some_html|e }}")
-    return template.render(some_html=content)
-

changedetectionio/model/App.py

@@ -56,11 +56,11 @@ class model(dict):
                     'rss_content_format': RSS_FORMAT_TYPES[0][0],
                     'rss_hide_muted_watches': True,
                     'rss_reader_mode': False,
-                    'scheduler_timezone_default': None,  # Default IANA timezone name
                     'schema_version' : 0,
                     'shared_diff_access': False,
                     'strip_ignored_lines': False,
                     'tags': {}, #@todo use Tag.model initialisers
+                    'timezone': None, # Default IANA timezone name
                     'webdriver_delay': None , # Extra delay in seconds before extracting text
                     'ui': {
                         'use_page_title_in_list': True,

changedetectionio/model/Watch.py

@@ -1,14 +1,14 @@
 from blinker import signal
 
 from changedetectionio.strtobool import strtobool
-from changedetectionio.jinja2_custom import render as jinja_render
+from changedetectionio.safe_jinja import render as jinja_render
 from . import watch_base
 import os
 import re
 from pathlib import Path
 from loguru import logger
 
-from .. import jinja2_custom as safe_jinja
+from .. import safe_jinja
 from ..html_tools import TRANSLATE_WHITESPACE_TABLE
 
 # Allowable protocols, protects against javascript: etc

changedetectionio/notification/__init__.py

@@ -1,5 +1,6 @@
 from changedetectionio.model import default_notification_format_for_watch
 
+ult_notification_format_for_watch = 'System default'
 default_notification_format = 'HTML Color'
 default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
 default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
@@ -15,3 +16,20 @@ valid_notification_formats = {
     default_notification_format_for_watch: default_notification_format_for_watch
 }
 
+
+valid_tokens = {
+    'base_url': '',
+    'current_snapshot': '',
+    'diff': '',
+    'diff_added': '',
+    'diff_full': '',
+    'diff_patch': '',
+    'diff_removed': '',
+    'diff_url': '',
+    'preview_url': '',
+    'triggered_text': '',
+    'watch_tag': '',
+    'watch_title': '',
+    'watch_url': '',
+    'watch_uuid': '',
+}

changedetectionio/notification/apprise_plugin/custom_handlers.py

@@ -70,7 +70,6 @@ def apprise_http_custom_handler(
     title: str,
     notify_type: str,
     meta: dict,
-    body_format: str = None,
     *args,
     **kwargs,
 ) -> bool:

changedetectionio/notification/handler.py

@@ -1,89 +1,18 @@
 
 import time
 import apprise
-from apprise import NotifyFormat
 from loguru import logger
-from urllib.parse import urlparse
 from .apprise_plugin.assets import apprise_asset, APPRISE_AVATAR_URL
-from .apprise_plugin.custom_handlers import SUPPORTED_HTTP_METHODS
-from ..notification_service import NotificationContextData
 
-
-def markup_text_links_to_html(body):
-    """
-    Convert plaintext to HTML with clickable links.
-    Uses Jinja2's escape and Markup for XSS safety.
-    """
-    from linkify_it import LinkifyIt
-    from markupsafe import Markup, escape
-
-    linkify = LinkifyIt()
-
-    # Match URLs in the ORIGINAL text (before escaping)
-    matches = linkify.match(body)
-
-    if not matches:
-        # No URLs, just escape everything
-        return Markup(escape(body))
-
-    result = []
-    last_index = 0
-
-    # Process each URL match
-    for match in matches:
-        # Add escaped text before the URL
-        if match.index > last_index:
-            text_part = body[last_index:match.index]
-            result.append(escape(text_part))
-
-        # Add the link with escaped URL (both in href and display)
-        url = match.url
-        result.append(Markup(f'<a href="{escape(url)}">{escape(url)}</a>'))
-
-        last_index = match.last_index
-
-    # Add remaining escaped text
-    if last_index < len(body):
-        result.append(escape(body[last_index:]))
-
-    # Join all parts
-    return str(Markup(''.join(str(part) for part in result)))
-
-def notification_format_align_with_apprise(n_format : str):
-    """
-    Correctly align changedetection's formats with apprise's formats
-    Probably these are the same - but good to be sure.
-    :param n_format:
-    :return:
-    """
-
-    if n_format.lower().startswith('html'):
-        # Apprise only knows 'html' not 'htmlcolor' etc, which shouldnt matter here
-        n_format = NotifyFormat.HTML.value
-    elif n_format.lower().startswith('markdown'):
-        # probably the same but just to be safe
-        n_format = NotifyFormat.MARKDOWN.value
-    elif n_format.lower().startswith('text'):
-        # probably the same but just to be safe
-        n_format = NotifyFormat.TEXT.value
-    else:
-        n_format = NotifyFormat.TEXT.value
-
-    return n_format
-
-def process_notification(n_object: NotificationContextData, datastore):
-    from changedetectionio.jinja2_custom import render as jinja_render
+def process_notification(n_object, datastore):
+    from changedetectionio.safe_jinja import render as jinja_render
     from . import default_notification_format_for_watch, default_notification_format, valid_notification_formats
     # be sure its registered
     from .apprise_plugin.custom_handlers import apprise_http_custom_handler
 
-    if not isinstance(n_object, NotificationContextData):
-        raise TypeError(f"Expected NotificationContextData, got {type(n_object)}")
-
     now = time.time()
     if n_object.get('notification_timestamp'):
         logger.trace(f"Time since queued {now-n_object['notification_timestamp']:.3f}s")
-
     # Insert variables into the notification content
     notification_parameters = create_notification_parameters(n_object, datastore)
 
@@ -95,9 +24,7 @@ def process_notification(n_object: NotificationContextData, datastore):
     # If we arrived with 'System default' then look it up
     if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
         # Initially text or whatever
-        n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format]).lower()
-
-    n_format = notification_format_align_with_apprise(n_format=n_format)
+        n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
 
     logger.trace(f"Complete notification body including Jinja and placeholders calculated in  {time.time() - now:.2f}s")
 
@@ -120,11 +47,7 @@ def process_notification(n_object: NotificationContextData, datastore):
 
             # Get the notification body from datastore
             n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
-
-            if n_object.get('markup_text_to_html'):
-                n_body = markup_text_links_to_html(body=n_body)
-
-            if n_format == NotifyFormat.HTML.value:
+            if n_object.get('notification_format', '').startswith('HTML'):
                 n_body = n_body.replace("\n", '<br>')
 
             n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
@@ -148,8 +71,7 @@ def process_notification(n_object: NotificationContextData, datastore):
             #     Length of URL - Incase they specify a longer custom avatar_url
 
             # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
-            parsed = urlparse(url)
-            k = '?' if not parsed.query else '&'
+            k = '?' if not '?' in url else '&'
             if not 'avatar_url' in url \
                     and not url.startswith('mail') \
                     and not url.startswith('post') \
@@ -178,15 +100,16 @@ def process_notification(n_object: NotificationContextData, datastore):
                 n_title = n_title[0:payload_max_size]
                 n_body = n_body[0:body_limit]
 
-            # Add format parameter to mailto URLs to ensure proper text/html handling
-            # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
-            # Note: Custom handlers (post://, get://, etc.) don't need this as we handle them
-            # differently by passing an invalid body_format to prevent HTML conversion
-            if not 'format=' in url and url.startswith(('mailto', 'mailtos')):
-                parsed = urlparse(url)
-                prefix = '?' if not parsed.query else '&'
-                # Apprise format is already lowercase from notification_format_align_with_apprise()
-                url = f"{url}{prefix}format={n_format}"
+            elif url.startswith('mailto'):
+                # Apprise will default to HTML, so we need to override it
+                # So that whats' generated in n_body is in line with what is going to be sent.
+                # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
+                if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
+                    prefix = '?' if not '?' in url else '&'
+                    # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
+                    n_format = n_format.lower()
+                    url = f"{url}{prefix}format={n_format}"
+                # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
 
             apobj.add(url)
 
@@ -196,28 +119,10 @@ def process_notification(n_object: NotificationContextData, datastore):
                               'body_format': n_format})
 
         # Blast off the notifications tht are set in .add()
-        # Check if we have any custom HTTP handlers (post://, get://, etc.)
-        # These handlers created with @notify decorator don't handle format conversion properly
-        # and will strip HTML if we pass a valid format. So we pass an invalid format string
-        # to prevent Apprise from converting HTML->TEXT
-
-        # Create list of custom handler protocols (both http and https versions)
-        custom_handler_protocols = [f"{method}://" for method in SUPPORTED_HTTP_METHODS]
-        custom_handler_protocols += [f"{method}s://" for method in SUPPORTED_HTTP_METHODS]
-
-        has_custom_handler = any(
-            url.startswith(tuple(custom_handler_protocols))
-            for url in n_object['notification_urls']
-        )
-
-        # If we have custom handlers, use invalid format to prevent conversion
-        # Otherwise use the proper format
-        notify_format = 'raw-no-convert' if has_custom_handler else n_format
-
         apobj.notify(
             title=n_title,
             body=n_body,
-            body_format=notify_format,
+            body_format=n_format,
             # False is not an option for AppRise, must be type None
             attach=n_object.get('screenshot', None)
         )
@@ -226,7 +131,7 @@ def process_notification(n_object: NotificationContextData, datastore):
         # Returns empty string if nothing found, multi-line string otherwise
         log_value = logs.getvalue()
 
-        if log_value and ('WARNING' in log_value or 'ERROR' in log_value):
+        if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
             logger.critical(log_value)
             raise Exception(log_value)
 
@@ -236,15 +141,17 @@ def process_notification(n_object: NotificationContextData, datastore):
 
 # Notification title + body content parameters get created here.
 # ( Where we prepare the tokens in the notification to be replaced with actual values )
-def create_notification_parameters(n_object: NotificationContextData, datastore):
-    if not isinstance(n_object, NotificationContextData):
-        raise TypeError(f"Expected NotificationContextData, got {type(n_object)}")
+def create_notification_parameters(n_object, datastore):
+    from copy import deepcopy
+    from . import valid_tokens
 
-    watch = datastore.data['watching'].get(n_object['uuid'])
-    if watch:
-        watch_title = datastore.data['watching'][n_object['uuid']].label
+    # in the case we send a test notification from the main settings, there is no UUID.
+    uuid = n_object['uuid'] if 'uuid' in n_object else ''
+
+    if uuid:
+        watch_title = datastore.data['watching'][uuid].label
         tag_list = []
-        tags = datastore.get_all_tags_for_watch(n_object['uuid'])
+        tags = datastore.get_all_tags_for_watch(uuid)
         if tags:
             for tag_uuid, tag in tags.items():
                 tag_list.append(tag.get('title'))
@@ -259,10 +166,14 @@ def create_notification_parameters(n_object: NotificationContextData, datastore)
 
     watch_url = n_object['watch_url']
 
-    diff_url = "{}/diff/{}".format(base_url, n_object['uuid'])
-    preview_url = "{}/preview/{}".format(base_url, n_object['uuid'])
+    diff_url = "{}/diff/{}".format(base_url, uuid)
+    preview_url = "{}/preview/{}".format(base_url, uuid)
 
-    n_object.update(
+    # Not sure deepcopy is needed here, but why not
+    tokens = deepcopy(valid_tokens)
+
+    # Valid_tokens also used as a field validator
+    tokens.update(
         {
             'base_url': base_url,
             'diff_url': diff_url,
@@ -270,10 +181,13 @@ def create_notification_parameters(n_object: NotificationContextData, datastore)
             'watch_tag': watch_tag if watch_tag is not None else '',
             'watch_title': watch_title if watch_title is not None else '',
             'watch_url': watch_url,
-            'watch_uuid': n_object['uuid'],
+            'watch_uuid': uuid,
         })
 
-    if watch:
-        n_object.update(datastore.data['watching'].get(n_object['uuid']).extra_notification_token_values())
+    # n_object will contain diff, diff_added etc etc
+    tokens.update(n_object)
 
-    return n_object
+    if uuid:
+        tokens.update(datastore.data['watching'].get(uuid).extra_notification_token_values())
+
+    return tokens

changedetectionio/notification_service.py

@@ -6,51 +6,9 @@ Extracted from update_worker.py to provide standalone notification functionality
 for both sync and async workers
 """
 
-from loguru import logger
 import time
+from loguru import logger
 
-from changedetectionio.notification import default_notification_format
-
-# What is passed around as notification context, also used as the complete list of valid {{ tokens }}
-class NotificationContextData(dict):
-    def __init__(self, initial_data=None, **kwargs):
-        super().__init__({
-            'current_snapshot': None,
-            'diff': None,
-            'diff_added': None,
-            'diff_full': None,
-            'diff_patch': None,
-            'diff_removed': None,
-            'notification_timestamp': time.time(),
-            'screenshot': None,
-            'triggered_text': None,
-            'uuid': 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX',  # Converted to 'watch_uuid' in create_notification_parameters
-            'watch_url': 'https://WATCH-PLACE-HOLDER/',
-            'base_url': None,
-            'diff_url': None,
-            'preview_url': None,
-            'watch_tag': None,
-            'watch_title': None,
-            'markup_text_to_html': False, # If automatic conversion of plaintext to HTML should happen
-        })
-
-        # Apply any initial data passed in
-        self.update({'watch_uuid': self.get('uuid')})
-        if initial_data:
-            self.update(initial_data)
-
-        # Apply any keyword arguments
-        if kwargs:
-            self.update(kwargs)
-
-    def set_random_for_validation(self):
-        import random, string
-        """Randomly fills all dict keys with random strings (for validation/testing)."""
-        for key in self.keys():
-            if key in ['uuid', 'time', 'watch_uuid']:
-                continue
-            rand_str = 'RANDOM-PLACEHOLDER-'+''.join(random.choices(string.ascii_letters + string.digits, k=12))
-            self[key] = rand_str
 
 class NotificationService:
     """
@@ -62,16 +20,13 @@ class NotificationService:
         self.datastore = datastore
         self.notification_q = notification_q
     
-    def queue_notification_for_watch(self, n_object: NotificationContextData, watch):
+    def queue_notification_for_watch(self, n_object, watch):
         """
         Queue a notification for a watch with full diff rendering and template variables
         """
         from changedetectionio import diff
         from changedetectionio.notification import default_notification_format_for_watch
 
-        if not isinstance(n_object, NotificationContextData):
-            raise TypeError(f"Expected NotificationContextData, got {type(n_object)}")
-
         dates = []
         trigger_text = ''
 
@@ -124,15 +79,15 @@ class NotificationService:
         n_object.update({
             'current_snapshot': snapshot_contents,
             'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
-            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
             'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
             'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
-            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
+            'notification_timestamp': now,
             'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
             'triggered_text': triggered_text,
             'uuid': watch.get('uuid') if watch else None,
             'watch_url': watch.get('url') if watch else None,
-            'watch_uuid': watch.get('uuid') if watch else None,
         })
 
         if watch:
@@ -185,7 +140,7 @@ class NotificationService:
         """
         Send notification when content changes are detected
         """
-        n_object = NotificationContextData()
+        n_object = {}
         watch = self.datastore.data['watching'].get(watch_uuid)
         if not watch:
             return
@@ -228,26 +183,11 @@ class NotificationService:
         if not watch:
             return
 
-        n_format = self.datastore.data['settings']['application'].get('notification_format', default_notification_format)
-        filter_list = ", ".join(watch['include_filters'])
-        # @todo - This could be a markdown template on the disk, apprise will convert the markdown to HTML+Plaintext parts in the email, and then 'markup_text_to_html' is not needed
-        body = f"""Hello,
-
-Your configured CSS/xPath filters of '{filter_list}' for {{{{watch_url}}}} did not appear on the page after {threshold} attempts.
-
-It's possible the page changed layout and the filter needs updating ( Try the 'Visual Selector' tab )
-
-Edit link: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}
-
-Thanks - Your omniscient changedetection.io installation.
-"""
-
-        n_object = NotificationContextData({
-            'notification_title': 'Changedetection.io - Alert - CSS/xPath filter was not present in the page',
-            'notification_body': body,
-            'notification_format': n_format,
-            'markup_text_to_html': n_format.lower().startswith('html')
-        })
+        n_object = {'notification_title': 'Changedetection.io - Alert - CSS/xPath filter was not present in the page',
+                    'notification_body': "Your configured CSS/xPath filters of '{}' for {{{{watch_url}}}} did not appear on the page after {} attempts, did the page change layout?\n\nLink: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\nThanks - Your omniscient changedetection.io installation :)\n".format(
+                        ", ".join(watch['include_filters']),
+                        threshold),
+                    'notification_format': 'text'}
 
         if len(watch['notification_urls']):
             n_object['notification_urls'] = watch['notification_urls']
@@ -275,28 +215,12 @@ Thanks - Your omniscient changedetection.io installation.
         if not watch:
             return
         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
-        n_format = self.datastore.data['settings']['application'].get('notification_format', default_notification_format).lower()
-        step = step_n + 1
-        # @todo - This could be a markdown template on the disk, apprise will convert the markdown to HTML+Plaintext parts in the email, and then 'markup_text_to_html' is not needed
-
-        # {{{{ }}}} because this will be Jinja2 {{ }} tokens
-        body = f"""Hello,
-        
-Your configured browser step at position {step} for the web page watch {{{{watch_url}}}} did not appear on the page after {threshold} attempts, did the page change layout?
-
-The element may have moved and needs editing, or does it need a delay added?
-
-Edit link: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}
-
-Thanks - Your omniscient changedetection.io installation.
-"""
-
-        n_object = NotificationContextData({
-            'notification_title': f"Changedetection.io - Alert - Browser step at position {step} could not be run",
-            'notification_body': body,
-            'notification_format': n_format,
-            'markup_text_to_html': n_format.lower().startswith('html')
-        })
+        n_object = {'notification_title': "Changedetection.io - Alert - Browser step at position {} could not be run".format(step_n+1),
+                    'notification_body': "Your configured browser step at position {} for {{{{watch_url}}}} "
+                                         "did not appear on the page after {} attempts, did the page change layout? "
+                                         "Does it need a delay added?\n\nLink: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\n"
+                                         "Thanks - Your omniscient changedetection.io installation :)\n".format(step_n+1, threshold),
+                    'notification_format': 'text'}
 
         if len(watch['notification_urls']):
             n_object['notification_urls'] = watch['notification_urls']

changedetectionio/processors/__init__.py

@@ -102,7 +102,7 @@ class difference_detection_processor():
             self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid'))
 
         # Tweak the base config with the per-watch ones
-        from changedetectionio.jinja2_custom import render as jinja_render
+        from changedetectionio.safe_jinja import render as jinja_render
         request_headers = CaseInsensitiveDict()
 
         ua = self.datastore.data['settings']['requests'].get('default_ua')

changedetectionio/processors/text_json_diff/processor.py

@@ -324,13 +324,13 @@ class ContentProcessor:
                     append_pretty_line_formatting=not self.watch.is_source_type_url
                 )
 
-        # Raise error if filter returned nothing
-        if not filtered_content.strip():
-            raise FilterNotFoundInResponse(
-                msg=self.filter_config.include_filters,
-                screenshot=self.fetcher.screenshot,
-                xpath_data=self.fetcher.xpath_data
-            )
+            # Raise error if filter returned nothing
+            if not filtered_content.strip():
+                raise FilterNotFoundInResponse(
+                    msg=self.filter_config.include_filters,
+                    screenshot=self.fetcher.screenshot,
+                    xpath_data=self.fetcher.xpath_data
+                )
 
         return filtered_content
 

changedetectionio/run_basic_tests.sh

@@ -15,7 +15,7 @@ find tests/test_*py -type f|while read test_name
 do
   echo "TEST RUNNING $test_name"
   # REMOVE_REQUESTS_OLD_SCREENSHOTS disabled so that we can write a screenshot and send it in test_notifications.py without a real browser
-  REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv -s --maxfail=1 --tb=long $test_name
+  REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest $test_name
 done
 
 echo "RUNNING WITH BASE_URL SET"
@@ -23,20 +23,20 @@ echo "RUNNING WITH BASE_URL SET"
 # Now re-run some tests with BASE_URL enabled
 # Re #65 - Ability to include a link back to the installation, in the notification.
 export BASE_URL="https://really-unique-domain.io"
-REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv -s --maxfail=1 tests/test_notification.py
+REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest tests/test_notification.py
 
 
 # Re-run with HIDE_REFERER set - could affect login
 export HIDE_REFERER=True
-pytest -vv -s --maxfail=1 tests/test_access_control.py
+pytest tests/test_access_control.py
 
 # Re-run a few tests that will trigger brotli based storage
 export SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD=5
-pytest -vv -s --maxfail=1 tests/test_access_control.py
+pytest tests/test_access_control.py
 REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest tests/test_notification.py
-pytest -vv -s --maxfail=1 tests/test_backend.py
-pytest -vv -s --maxfail=1 tests/test_rss.py
-pytest -vv -s --maxfail=1 tests/test_unique_lines.py
+pytest tests/test_backend.py
+pytest tests/test_rss.py
+pytest tests/test_unique_lines.py
 
 # Try high concurrency
 FETCH_WORKERS=130 pytest  tests/test_history_consistency.py -v -l

changedetectionio/safe_jinja.py

@@ -0,0 +1,24 @@
+"""
+Safe Jinja2 render with max payload sizes
+
+See https://jinja.palletsprojects.com/en/3.1.x/sandbox/#security-considerations
+"""
+
+import jinja2.sandbox
+import typing as t
+import os
+
+JINJA2_MAX_RETURN_PAYLOAD_SIZE = 1024 * int(os.getenv("JINJA2_MAX_RETURN_PAYLOAD_SIZE_KB", 1024 * 10))
+
+# This is used for notifications etc, so actually it's OK to send custom HTML such as <a href> etc, but it should limit what data is available.
+# (Which also limits available functions that could be called)
+def render(template_str, **args: t.Any) -> str:
+    jinja2_env = jinja2.sandbox.ImmutableSandboxedEnvironment(extensions=['jinja2_time.TimeExtension'])
+    output = jinja2_env.from_string(template_str).render(args)
+    return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]
+
+def render_fully_escaped(content):
+    env = jinja2.sandbox.ImmutableSandboxedEnvironment(autoescape=True)
+    template = env.from_string("{{ some_html|e }}")
+    return template.render(some_html=content)
+

changedetectionio/static/js/global-settings.js

@@ -29,7 +29,7 @@ $(document).ready(function () {
         $(this).text(new Date($(this).data("utc")).toLocaleString());
     })
 
-    const timezoneInput = $('#application-scheduler_timezone_default');
+    const timezoneInput = $('#application-timezone');
     if(timezoneInput.length) {
         const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
         if (!timezoneInput.val().trim()) {

changedetectionio/store.py

@@ -228,36 +228,26 @@ class ChangeDetectionStore:
         d['settings']['application']['active_base_url'] = active_base_url.strip('" ')
         return d
 
-    from pathlib import Path
-
-    def delete_path(self, path: Path):
-        import shutil
-        """Delete a file or directory tree, including the path itself."""
-        if not path.exists():
-            return
-        if path.is_file() or path.is_symlink():
-            path.unlink(missing_ok=True)  # deletes a file or symlink
-        else:
-            shutil.rmtree(path, ignore_errors=True)  # deletes dir *and* its contents
-
     # Delete a single watch by UUID
     def delete(self, uuid):
         import pathlib
+        import shutil
 
         with self.lock:
             if uuid == 'all':
                 self.__data['watching'] = {}
                 time.sleep(1) # Mainly used for testing to allow all items to flush before running next test
+
+                # GitHub #30 also delete history records
                 for uuid in self.data['watching']:
                     path = pathlib.Path(os.path.join(self.datastore_path, uuid))
                     if os.path.exists(path):
-                        self.delete(uuid)
+                        shutil.rmtree(path)
 
             else:
                 path = pathlib.Path(os.path.join(self.datastore_path, uuid))
                 if os.path.exists(path):
-                    self.delete_path(path)
-
+                    shutil.rmtree(path)
                 del self.data['watching'][uuid]
 
         self.needs_write_urgent = True
@@ -986,10 +976,6 @@ class ChangeDetectionStore:
         if self.data['settings']['application'].get('extract_title_as_title'):
             self.data['settings']['application']['ui']['use_page_title_in_list'] = self.data['settings']['application'].get('extract_title_as_title')
 
-    def update_21(self):
-        self.data['settings']['application']['scheduler_timezone_default'] = self.data['settings']['application'].get('timezone')
-        del self.data['settings']['application']['timezone']
-
 
     def add_notification_url(self, notification_url):
         

changedetectionio/templates/_helpers.html

@@ -14,31 +14,13 @@
                 {% if field.errors is mapping and 'form' in field.errors %}
                     {#  and subfield form errors, such as used in RequiredFormField() for TimeBetweenCheckForm sub form #}
                     {% set errors = field.errors['form'] %}
-                    {% for error in errors %}
-                        <li>{{ error }}</li>
-                    {% endfor %}
-                {% elif field.type == 'FieldList' %}
-                    {# Handle FieldList of FormFields - errors is a list of dicts, one per entry #}
-                    {% for idx, entry_errors in field.errors|enumerate %}
-                        {% if entry_errors is mapping and entry_errors %}
-                            {# Only show entries that have actual errors #}
-                            <li><strong>Entry {{ idx + 1 }}:</strong>
-                                <ul>
-                                    {% for field_name, messages in entry_errors.items() %}
-                                        {% for message in messages %}
-                                            <li>{{ field_name }}: {{ message }}</li>
-                                        {% endfor %}
-                                    {% endfor %}
-                                </ul>
-                            </li>
-                        {% endif %}
-                    {% endfor %}
                 {% else %}
                     {#  regular list of errors with this field #}
-                    {% for error in field.errors %}
-                        <li>{{ error }}</li>
-                    {% endfor %}
+                    {% set errors = field.errors %}
                 {% endif %}
+                {% for error in errors %}
+                    <li>{{ error }}</li>
+                {% endfor %}
             </ul>
         {% endif %}
     </div>
@@ -111,39 +93,6 @@
   {{ field(**kwargs)|safe }}
 {% endmacro %}
 
-{% macro render_fieldlist_with_inline_errors(fieldlist) %}
-  {# Specialized macro for FieldList(FormField(...)) that renders errors inline with each field #}
-  <div {% if fieldlist.errors %} class="error" {% endif %}>{{ fieldlist.label }}</div>
-  <div {% if fieldlist.errors %} class="error" {% endif %}>
-    <ul id="{{ fieldlist.id }}">
-      {% for entry in fieldlist %}
-        <li {% if entry.errors %} class="error" {% endif %}>
-          <label for="{{ entry.id }}" {% if entry.errors %} class="error" {% endif %}>{{ fieldlist.label.text }}-{{ loop.index0 }}</label>
-          <table id="{{ entry.id }}" {% if entry.errors %} class="error" {% endif %}>
-            <tbody>
-              {% for subfield in entry %}
-                <tr {% if subfield.errors %} class="error" {% endif %}>
-                  <th {% if subfield.errors %} class="error" {% endif %}><label for="{{ subfield.id }}" {% if subfield.errors %} class="error" {% endif %}>{{ subfield.label.text }}</label></th>
-                  <td {% if subfield.errors %} class="error" {% endif %}>
-                    {{ subfield(**kwargs)|safe }}
-                    {% if subfield.errors %}
-                      <ul class="errors">
-                        {% for error in subfield.errors %}
-                          <li class="error">{{ error }}</li>
-                        {% endfor %}
-                      </ul>
-                    {% endif %}
-                  </td>
-                </tr>
-              {% endfor %}
-            </tbody>
-          </table>
-        </li>
-      {% endfor %}
-    </ul>
-  </div>
-{% endmacro %}
-
 {% macro render_conditions_fieldlist_of_formfields_as_table(fieldlist, table_id="rulesTable") %}
   <div class="fieldlist_formfields" id="{{ table_id }}">
     <div class="fieldlist-header">

changedetectionio/tests/conftest.py

@@ -4,7 +4,6 @@ import time
 from threading import Thread
 
 import pytest
-import arrow
 from changedetectionio import changedetection_app
 from changedetectionio import store
 import os
@@ -30,17 +29,6 @@ def reportlog(pytestconfig):
     logger.remove(handler_id)
 
 
-@pytest.fixture
-def environment(mocker):
-    """Mock arrow.now() to return a fixed datetime for testing jinja2 time extension."""
-    # Fixed datetime: Wed, 09 Dec 2015 23:33:01 UTC
-    # This is calculated to match the test expectations when offsets are applied
-    fixed_datetime = arrow.Arrow(2015, 12, 9, 23, 33, 1, tzinfo='UTC')
-    # Patch arrow.now in the TimeExtension module where it's actually used
-    mocker.patch('changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now', return_value=fixed_datetime)
-    return fixed_datetime
-
-
 def format_memory_human(bytes_value):
     """Format memory in human-readable units (KB, MB, GB)"""
     if bytes_value < 1024:

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -49,39 +49,3 @@ def test_select_custom(client, live_server, measure_memory_usage):
     #
     # Now we should see the request in the container logs for "squid-squid-custom" because it will be the only default
 
-
-def test_custom_proxy_validation(client, live_server, measure_memory_usage):
-    #  live_server_setup(live_server) # Setup on conftest per function
-
-    # Goto settings, add our custom one
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={
-            "requests-time_between_check-minutes": 180,
-            "application-ignore_whitespace": "y",
-            "application-fetch_backend": 'html_requests',
-            "requests-extra_proxies-0-proxy_name": "custom-test-proxy",
-            "requests-extra_proxies-0-proxy_url": "xxxxhtt/333??p://test:awesome@squid-custom:3128",
-        },
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." not in res.data
-    assert b'Proxy URLs must start with' in res.data
-
-
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={
-            "requests-time_between_check-minutes": 180,
-            "application-ignore_whitespace": "y",
-            "application-fetch_backend": 'html_requests',
-            "requests-extra_proxies-0-proxy_name": "custom-test-proxy",
-            "requests-extra_proxies-0-proxy_url": "https://",
-        },
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." not in res.data
-    assert b"Invalid URL." in res.data
-    

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -3,10 +3,6 @@ import os
 import time
 import re
 from flask import url_for
-from email import message_from_string
-from email.policy import default as email_policy
-
-from changedetectionio.diff import REMOVED_STYLE, ADDED_STYLE
 from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
     wait_for_all_checks, \
     set_longer_modified_response, delete_all_watches
@@ -54,7 +50,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
         url_for("settings.settings_page"),
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": "some text\nfallback-body<br> " + default_notification_body,
+              "application-notification_body": "fallback-body<br> " + default_notification_body,
               "application-notification_format": 'HTML',
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
@@ -81,164 +77,14 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
 
     time.sleep(3)
 
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
 
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
-
-    # The email should have two bodies (multipart/alternative with text/plain and text/html)
-    assert msg.is_multipart()
-    assert msg.get_content_type() == 'multipart/alternative'
-
-    # Get the parts
-    parts = list(msg.iter_parts())
-    assert len(parts) == 2
-
-    # First part should be text/plain (the auto-generated plaintext version)
-    text_part = parts[0]
-    assert text_part.get_content_type() == 'text/plain'
-    text_content = text_part.get_content()
-    assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
-    assert 'fallback-body\r\n' in text_content  # The plaintext part
-
-    # Second part should be text/html
-    html_part = parts[1]
-    assert html_part.get_content_type() == 'text/html'
-    html_content = html_part.get_content()
-    assert 'some text<br>' in html_content  # We converted \n from the notification body
-    assert 'fallback-body<br>' in html_content  # kept the original <br>
-    assert '(added) So let\'s see what happens.<br>' in html_content  # the html part
-    delete_all_watches(client)
-
-
-def test_check_notification_plaintext_format(client, live_server, measure_memory_usage):
-    set_original_response()
-
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": "some text\n" + default_notification_body,
-              "application-notification_format": 'Text',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    # Add a watch and trigger a HTTP POST
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("ui.ui_views.form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    wait_for_all_checks(client)
-    set_longer_modified_response()
-    time.sleep(2)
-
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
-
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
-
-    # The email should be plain text only (not multipart)
-    assert not msg.is_multipart()
-    assert msg.get_content_type() == 'text/plain'
-
-    # Get the plain text content
-    text_content = msg.get_content()
-    assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
-
-    # Should NOT contain HTML
-    assert '<br>' not in text_content  # We should not have HTML in plain text
-    delete_all_watches(client)
-
-
-
-def test_check_notification_html_color_format(client, live_server, measure_memory_usage):
-    set_original_response()
-
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": "some text\n" + default_notification_body,
-              "application-notification_format": 'HTML Color',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    # Add a watch and trigger a HTTP POST
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("ui.ui_views.form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    wait_for_all_checks(client)
-    set_longer_modified_response()
-    time.sleep(2)
-
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
-
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
-
-    # The email should have two bodies (multipart/alternative with text/plain and text/html)
-    assert msg.is_multipart()
-    assert msg.get_content_type() == 'multipart/alternative'
-
-    # Get the parts
-    parts = list(msg.iter_parts())
-    assert len(parts) == 2
-
-    # First part should be text/plain (the auto-generated plaintext version)
-    text_part = parts[0]
-    assert text_part.get_content_type() == 'text/plain'
-    text_content = text_part.get_content()
-    assert 'So let\'s see what happens.\r\n' in text_content  # The plaintext part
-    assert '(added)' not in text_content # Because apprise only dumb converts the html to text
-
-    # Second part should be text/html with color styling
-    html_part = parts[1]
-    assert html_part.get_content_type() == 'text/html'
-    html_content = html_part.get_content()
-    assert REMOVED_STYLE in html_content
-    assert ADDED_STYLE in html_content
-
-    assert 'some text<br>' in html_content
+    # The email should have two bodies, and the text/html part should be <br>
+    assert 'Content-Type: text/plain' in msg
+    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
+    assert 'Content-Type: text/html' in msg
+    assert '(added) So let\'s see what happens.<br>' in msg  # the html part
     delete_all_watches(client)
 
 
@@ -293,21 +139,15 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     wait_for_all_checks(client)
 
     time.sleep(3)
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
     #    with open('/tmp/m.txt', 'w') as f:
-    #        f.write(msg_raw)
-
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
+    #        f.write(msg)
 
     # The email should not have two bodies, should be TEXT only
-    assert not msg.is_multipart()
-    assert msg.get_content_type() == 'text/plain'
 
-    # Get the plain text content
-    text_content = msg.get_content()
-    assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
+    assert 'Content-Type: text/plain' in msg
+    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
 
     set_original_response()
     # Now override as HTML format
@@ -324,34 +164,18 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     wait_for_all_checks(client)
 
     time.sleep(3)
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
 
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
-
-    # The email should have two bodies (multipart/alternative)
-    assert msg.is_multipart()
-    assert msg.get_content_type() == 'multipart/alternative'
-
-    # Get the parts
-    parts = list(msg.iter_parts())
-    assert len(parts) == 2
-
-    # First part should be text/plain
-    text_part = parts[0]
-    assert text_part.get_content_type() == 'text/plain'
-    text_content = text_part.get_content()
-    assert '(removed) So let\'s see what happens.\r\n' in text_content  # The plaintext part
-
-    # Second part should be text/html
-    html_part = parts[1]
-    assert html_part.get_content_type() == 'text/html'
-    html_content = html_part.get_content()
-    assert '(removed) So let\'s see what happens.<br>' in html_content  # the html part
+    # The email should have two bodies, and the text/html part should be <br>
+    assert 'Content-Type: text/plain' in msg
+    assert '(removed) So let\'s see what happens.\r\n' in msg  # The plaintext part with \n
+    assert 'Content-Type: text/html' in msg
+    assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
 
     # https://github.com/dgtlmoon/changedetection.io/issues/2103
-    assert '<h1>Test</h1>' in html_content
-    assert '&lt;' not in html_content
+    assert '<h1>Test</h1>' in msg
+    assert '&lt;' not in msg
+    assert 'Content-Type: text/html' in msg
 
     delete_all_watches(client)

changedetectionio/tests/test_backend.py

@@ -165,46 +165,6 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # Cleanup everything
     delete_all_watches(client)
 
-
-# Server says its plaintext, we should always treat it as plaintext, and then if they have a filter, try to apply that
-def test_requests_timeout(client, live_server, measure_memory_usage):
-    delay = 2
-    test_url = url_for('test_endpoint', delay=delay, _external=True)
-
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-ui-use_page_title_in_list": "",
-              "requests-time_between_check-minutes": 180,
-              "requests-timeout": delay - 1,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    # Add our URL to the import page
-    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    # requests takes >2 sec but we timeout at 1 second
-    res = client.get(url_for("watchlist.index"))
-    assert b'Read timed out. (read timeout=1)' in res.data
-
-    ##### Now set a longer timeout
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-ui-use_page_title_in_list": "",
-              "requests-time_between_check-minutes": 180,
-              "requests-timeout": delay + 1, # timeout should be a second more than the reply time
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-
-    wait_for_all_checks(client)
-
-    res = client.get(url_for("watchlist.index"))
-    assert b'Read timed out' not in res.data
-
 def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage):
     """
 
@@ -378,4 +338,4 @@ def test_plaintext_even_if_xml_content_and_can_apply_filters(client, live_server
     assert b'<string name="feed_update_receiver_name"' in res.data
     assert b'<foobar' not in res.data
 
-    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_filter_exist_changes.py

@@ -93,9 +93,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
         "tags": "my tag",
         "title": "my title",
         "headers": "",
-        # preprended with extra filter that intentionally doesn't match any entry,
-        # notification should still be sent even if first filter does not match (PR#3516)
-        "include_filters": ".non-matching-selector\n.ticket-available",
+        "include_filters": '.ticket-available',
         "fetch_backend": "html_requests",
         "time_between_check_use_default": "y"})
 

changedetectionio/tests/test_filter_failure_notification.py

@@ -1,8 +1,10 @@
 import os
 import time
+from loguru import logger
 from flask import url_for
-from .util import set_original_response,  wait_for_all_checks, wait_for_notification_endpoint_output
-from ..notification import valid_notification_formats
+from .util import set_original_response, live_server_setup, extract_UUID_from_client, wait_for_all_checks, \
+    wait_for_notification_endpoint_output
+from changedetectionio.model import App
 
 
 def set_response_with_filter():
@@ -21,14 +23,13 @@ def set_response_with_filter():
         f.write(test_return_data)
     return None
 
-def run_filter_test(client, live_server, content_filter, app_notification_format):
+def run_filter_test(client, live_server, content_filter):
 
     # Response WITHOUT the filter ID element
     set_original_response()
-    live_server.app.config['DATASTORE'].data['settings']['application']['notification_format'] = app_notification_format
 
     # Goto the edit page, add our ignore text
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'post')
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -126,23 +127,8 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
     with open("test-datastore/notification.txt", 'r') as f:
         notification = f.read()
 
-    assert 'Your configured CSS/xPath filters' in notification
-
-
-    # Text (or HTML conversion) markup to make the notifications a little nicer should have worked
-    if app_notification_format.startswith('html'):
-        # apprise should have used sax-escape (' instead of ", " etc), lets check it worked
-
-        from apprise.conversion import convert_between
-        from apprise.common import NotifyFormat
-        escaped_filter = convert_between(NotifyFormat.TEXT, NotifyFormat.HTML, content_filter)
-
-        assert escaped_filter in notification or escaped_filter.replace('"', '"') in notification
-        assert 'a href="' in notification # Quotes should still be there so the link works
-
-    else:
-        assert 'a href' not in notification
-        assert content_filter in notification
+    assert 'CSS/xPath filter was not present in the page' in notification
+    assert content_filter.replace('"', '\\"') in notification
 
     # Remove it and prove that it doesn't trigger when not expected
     # It should register a change, but no 'filter not found'
@@ -173,20 +159,14 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
     os.unlink("test-datastore/notification.txt")
 
 
+
+
 def test_check_include_filters_failure_notification(client, live_server, measure_memory_usage):
-    #   #  live_server_setup(live_server) # Setup on conftest per function
-    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('HTML Color'))
-    # Check markup send conversion didnt affect plaintext preference
-    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('Text'))
+#   #  live_server_setup(live_server) # Setup on conftest per function
+    run_filter_test(client, live_server,'#nope-doesnt-exist')
 
 def test_check_xpath_filter_failure_notification(client, live_server, measure_memory_usage):
-    #   #  live_server_setup(live_server) # Setup on conftest per function
-    run_filter_test(client=client, live_server=live_server, content_filter='//*[@id="nope-doesnt-exist"]', app_notification_format=valid_notification_formats.get('HTML Color'))
+#   #  live_server_setup(live_server) # Setup on conftest per function
+    run_filter_test(client, live_server, '//*[@id="nope-doesnt-exist"]')
 
 # Test that notification is never sent
-
-def test_basic_markup_from_text(client, live_server, measure_memory_usage):
-    # Test the notification error templates convert to HTML if needed (link activate)
-    from ..notification.handler import markup_text_links_to_html
-    x = markup_text_links_to_html("hello https://google.com")
-    assert 'a href' in x

changedetectionio/tests/test_jinja2.py

@@ -1,10 +1,8 @@
 #!/usr/bin/env python3
 
 import time
-import arrow
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
-from ..jinja2_custom import render
 
 
 # def test_setup(client, live_server, measure_memory_usage):
@@ -35,35 +33,6 @@ def test_jinja2_in_url_query(client, live_server, measure_memory_usage):
     )
     assert b'date=2' in res.data
 
-# Test for issue #1493 - jinja2-time offset functionality
-def test_jinja2_time_offset_in_url_query(client, live_server, measure_memory_usage):
-    """Test that jinja2 time offset expressions work in watch URLs (issue #1493)."""
-
-    # Add our URL to the import page with time offset expression
-    test_url = url_for('test_return_query', _external=True)
-
-    # Test the exact syntax from issue #1493 that was broken in jinja2-time
-    # This should work now with our custom TimeExtension
-    full_url = "{}?{}".format(test_url,
-                              "timestamp={% now 'utc' - 'minutes=11', '%Y-%m-%d %H:%M' %}", )
-    res = client.post(
-        url_for("ui.ui_views.form_quick_watch_add"),
-        data={"url": full_url, "tags": "test"},
-        follow_redirects=True
-    )
-    assert b"Watch added" in res.data
-    wait_for_all_checks(client)
-
-    # Verify the URL was processed correctly (should not have errors)
-    res = client.get(
-        url_for("ui.ui_views.preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    # Should have a valid timestamp in the response
-    assert b'timestamp=' in res.data
-    # Should not have template error
-    assert b'Invalid template' not in res.data
-
 # https://techtonics.medium.com/secure-templating-with-jinja2-understanding-ssti-and-jinja2-sandbox-environment-b956edd60456
 def test_jinja2_security_url_query(client, live_server, measure_memory_usage):
     
@@ -87,86 +56,3 @@ def test_jinja2_security_url_query(client, live_server, measure_memory_usage):
     assert b'is invalid and cannot be used' in res.data
     # Some of the spewed output from the subclasses
     assert b'dict_values' not in res.data
-
-def test_timezone(mocker):
-    """Verify that timezone is parsed."""
-
-    timezone = 'America/Buenos_Aires'
-    currentDate = arrow.now(timezone)
-    arrowNowMock = mocker.patch("changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now")
-    arrowNowMock.return_value = currentDate
-    finalRender = render(f"{{% now '{timezone}' %}}")
-
-    assert finalRender == currentDate.strftime('%a, %d %b %Y %H:%M:%S')
-
-def test_format(mocker):
-    """Verify that format is parsed."""
-
-    timezone = 'utc'
-    format = '%d %b %Y %H:%M:%S'
-    currentDate = arrow.now(timezone)
-    arrowNowMock = mocker.patch("arrow.now")
-    arrowNowMock.return_value = currentDate
-    finalRender = render(f"{{% now '{timezone}', '{format}' %}}")
-
-    assert finalRender == currentDate.strftime(format)
-
-def test_add_time(environment):
-    """Verify that added time offset can be parsed."""
-
-    finalRender = render("{% now 'utc' + 'hours=2,seconds=30' %}")
-
-    assert finalRender == "Thu, 10 Dec 2015 01:33:31"
-
-def test_add_weekday(mocker):
-    """Verify that added weekday offset can be parsed."""
-
-    timezone = 'utc'
-    currentDate = arrow.now(timezone)
-    arrowNowMock = mocker.patch("changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now")
-    arrowNowMock.return_value = currentDate
-    finalRender = render(f"{{% now '{timezone}' + 'weekday=1' %}}")
-
-    assert finalRender == currentDate.shift(weekday=1).strftime('%a, %d %b %Y %H:%M:%S')
-
-
-def test_substract_time(environment):
-    """Verify that substracted time offset can be parsed."""
-
-    finalRender = render("{% now 'utc' - 'minutes=11' %}")
-
-    assert finalRender == "Wed, 09 Dec 2015 23:22:01"
-
-
-def test_offset_with_format(environment):
-    """Verify that offset works together with datetime format."""
-
-    finalRender = render(
-        "{% now 'utc' - 'days=2,minutes=33,seconds=1', '%d %b %Y %H:%M:%S' %}"
-    )
-
-    assert finalRender == "07 Dec 2015 23:00:00"
-
-def test_default_timezone_empty_string(environment):
-    """Verify that empty timezone string uses the default timezone (UTC in test environment)."""
-
-    # Empty string should use the default timezone which is 'UTC' (or from application settings)
-    finalRender = render("{% now '' %}")
-
-    # Should render with default format and UTC timezone (matches environment fixture)
-    assert finalRender == "Wed, 09 Dec 2015 23:33:01"
-
-def test_default_timezone_with_offset(environment):
-    """Verify that empty timezone works with offset operations."""
-
-    # Empty string with offset should use default timezone
-    finalRender = render("{% now '' + 'hours=2', '%d %b %Y %H:%M:%S' %}")
-
-    assert finalRender == "10 Dec 2015 01:33:01"
-
-def test_default_timezone_subtraction(environment):
-    """Verify that empty timezone works with subtraction offset."""
-
-    finalRender = render("{% now '' - 'minutes=11' %}")
-
-    assert finalRender == "Wed, 09 Dec 2015 23:22:01"

changedetectionio/tests/test_notification.py

@@ -284,27 +284,6 @@ def test_notification_validation(client, live_server, measure_memory_usage):
     )
 
 
-def test_notification_urls_jinja2_apprise_integration(client, live_server, measure_memory_usage):
-
-    #
-    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = "hassio://127.0.0.1/longaccesstoken?verify=no&nid={{watch_uuid}}"
-
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={
-              "application-fetch_backend": "html_requests",
-              "application-minutes_between_check": 180,
-              "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444, "somebug": "网站监测 内容更新了" }',
-              "application-notification_format": default_notification_format,
-              "application-notification_urls": test_notification_url,
-              # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-              "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }} ",
-              },
-        follow_redirects=True
-    )
-    assert b'Settings updated' in res.data
-
 
 def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_memory_usage):
     
@@ -315,7 +294,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&watch_uuid={{ watch_uuid }}&xxx={{ watch_url }}&now={% now 'Europe/London', '%Y-%m-%d' %}&+custom-header=123&+second=hello+world%20%22space%22"
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
 
     res = client.post(
         url_for("settings.settings_page"),
@@ -341,7 +320,6 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     )
 
     assert b"Watch added" in res.data
-    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     wait_for_all_checks(client)
     set_modified_response()
@@ -371,11 +349,6 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
         assert 'xxx=http' in notification_url
         # apprise style headers should be stripped
         assert 'custom-header' not in notification_url
-        # Check jinja2 custom arrow/jinja2-time replace worked
-        assert 'now=2' in notification_url
-        # Check our watch_uuid appeared
-        assert f'watch_uuid={watch_uuid}' in notification_url
-
 
     with open("test-datastore/notification-headers.txt", 'r') as f:
         notification_headers = f.read()
@@ -443,6 +416,7 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert res.status_code != 400
     assert res.status_code != 500
 
+
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
         assert test_body in x
@@ -541,7 +515,9 @@ def _test_color_notifications(client, notification_body_token):
         follow_redirects=True
     )
 
-# Just checks the format of the colour notifications was correct
 def test_html_color_notifications(client, live_server, measure_memory_usage):
+
+    
     _test_color_notifications(client, '{{diff}}')
     _test_color_notifications(client, '{{diff_full}}')
+    

changedetectionio/tests/test_scheduler.py

@@ -24,7 +24,7 @@ def test_check_basic_scheduler_functionality(client, live_server, measure_memory
         url_for("settings.settings_page"),
         data={"application-empty_pages_are_a_change": "",
               "requests-time_between_check-seconds": 1,
-              "application-scheduler_timezone_default": "Pacific/Kiritimati",  # Most Forward Time Zone (UTC+14:00)
+              "application-timezone": "Pacific/Kiritimati",  # Most Forward Time Zone (UTC+14:00)
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
     )
@@ -119,7 +119,7 @@ def test_check_basic_global_scheduler_functionality(client, live_server, measure
 
     data = {
         "application-empty_pages_are_a_change": "",
-        "application-scheduler_timezone_default": "Pacific/Kiritimati",  # Most Forward Time Zone (UTC+14:00)
+        "application-timezone": "Pacific/Kiritimati",  # Most Forward Time Zone (UTC+14:00)
         'application-fetch_backend': "html_requests",
         "requests-time_between_check-hours": 0,
         "requests-time_between_check-minutes": 0,

changedetectionio/tests/unit/test_jinja2_security.py

@@ -4,7 +4,7 @@
 # python3 -m unittest changedetectionio.tests.unit.test_jinja2_security
 
 import unittest
-from changedetectionio import jinja2_custom as safe_jinja
+from changedetectionio import safe_jinja
 
 
 # mostly

changedetectionio/tests/unit/test_restock_logic.py

@@ -6,7 +6,7 @@
 import unittest
 import os
 
-import changedetectionio.processors.restock_diff.processor as restock_diff
+from changedetectionio.processors import restock_diff
 
 # mostly
 class TestDiffBuilder(unittest.TestCase):

changedetectionio/tests/unit/test_scheduler.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 
 # run from dir above changedetectionio/ dir
-# python3 -m unittest changedetectionio.tests.unit.test_scheduler
+# python3 -m unittest changedetectionio.tests.unit.test_jinja2_security
 
 import unittest
-import arrow
+from datetime import datetime, timedelta
+from zoneinfo import ZoneInfo
 
 class TestScheduler(unittest.TestCase):
 
@@ -12,13 +13,12 @@ class TestScheduler(unittest.TestCase):
     # UTC-12:00 (Baker Island, Howland Island) is the farthest behind, always one calendar day behind UTC.
 
     def test_timezone_basic_time_within_schedule(self):
-        """Test that current time is detected as within schedule window."""
         from changedetectionio import time_handler
 
         timezone_str = 'Europe/Berlin'
-        debug_datetime = arrow.now(timezone_str)
-        day_of_week = debug_datetime.format('dddd')
-        time_str = debug_datetime.format('HH:00')
+        debug_datetime = datetime.now(ZoneInfo(timezone_str))
+        day_of_week = debug_datetime.strftime('%A')
+        time_str = str(debug_datetime.hour)+':00'
         duration = 60  # minutes
 
         # The current time should always be within 60 minutes of [time_hour]:00
@@ -30,17 +30,16 @@ class TestScheduler(unittest.TestCase):
         self.assertEqual(result, True, f"{debug_datetime} is within time scheduler {day_of_week} {time_str} in {timezone_str} for {duration} minutes")
 
     def test_timezone_basic_time_outside_schedule(self):
-        """Test that time from yesterday is outside current schedule."""
         from changedetectionio import time_handler
 
         timezone_str = 'Europe/Berlin'
-        # We try a date in the past (yesterday)
-        debug_datetime = arrow.now(timezone_str).shift(days=-1)
-        day_of_week = debug_datetime.format('dddd')
-        time_str = debug_datetime.format('HH:00')
-        duration = 60 * 24  # minutes
+        # We try a date in the future..
+        debug_datetime = datetime.now(ZoneInfo(timezone_str))+ timedelta(days=-1)
+        day_of_week = debug_datetime.strftime('%A')
+        time_str = str(debug_datetime.hour) + ':00'
+        duration = 60*24  # minutes
 
-        # The current time should NOT be within yesterday's schedule
+        # The current time should always be within 60 minutes of [time_hour]:00
         result = time_handler.am_i_inside_time(day_of_week=day_of_week,
                                                time_str=time_str,
                                                timezone_str=timezone_str,
@@ -49,58 +48,6 @@ class TestScheduler(unittest.TestCase):
         self.assertNotEqual(result, True,
                          f"{debug_datetime} is NOT within time scheduler {day_of_week} {time_str} in {timezone_str} for {duration} minutes")
 
-    def test_timezone_utc_within_schedule(self):
-        """Test UTC timezone works correctly."""
-        from changedetectionio import time_handler
-
-        timezone_str = 'UTC'
-        debug_datetime = arrow.now(timezone_str)
-        day_of_week = debug_datetime.format('dddd')
-        time_str = debug_datetime.format('HH:00')
-        duration = 120  # minutes
-
-        result = time_handler.am_i_inside_time(day_of_week=day_of_week,
-                                               time_str=time_str,
-                                               timezone_str=timezone_str,
-                                               duration=duration)
-
-        self.assertTrue(result, "Current time should be within UTC schedule")
-
-    def test_timezone_extreme_ahead(self):
-        """Test with UTC+14 timezone (Line Islands, Kiribati)."""
-        from changedetectionio import time_handler
-
-        timezone_str = 'Pacific/Kiritimati'  # UTC+14
-        debug_datetime = arrow.now(timezone_str)
-        day_of_week = debug_datetime.format('dddd')
-        time_str = debug_datetime.format('HH:00')
-        duration = 60
-
-        result = time_handler.am_i_inside_time(day_of_week=day_of_week,
-                                               time_str=time_str,
-                                               timezone_str=timezone_str,
-                                               duration=duration)
-
-        self.assertTrue(result, "Should work with extreme ahead timezone")
-
-    def test_timezone_extreme_behind(self):
-        """Test with UTC-12 timezone (Baker Island)."""
-        from changedetectionio import time_handler
-
-        # Using Etc/GMT+12 which is UTC-12 (confusing, but that's how it works)
-        timezone_str = 'Etc/GMT+12'  # UTC-12
-        debug_datetime = arrow.now(timezone_str)
-        day_of_week = debug_datetime.format('dddd')
-        time_str = debug_datetime.format('HH:00')
-        duration = 60
-
-        result = time_handler.am_i_inside_time(day_of_week=day_of_week,
-                                               time_str=time_str,
-                                               timezone_str=timezone_str,
-                                               duration=duration)
-
-        self.assertTrue(result, "Should work with extreme behind timezone")
-
 
 if __name__ == '__main__':
     unittest.main()

changedetectionio/tests/unit/test_time_extension.py

@@ -1,138 +0,0 @@
-#!/usr/bin/env python3
-"""
-Simple unit tests for TimeExtension that mimic how safe_jinja.py uses it.
-These tests demonstrate that the environment.default_timezone override works
-exactly as intended in the actual application code.
-"""
-
-import arrow
-from jinja2.sandbox import ImmutableSandboxedEnvironment
-from changedetectionio.jinja2_custom.extensions.TimeExtension import TimeExtension
-
-
-def test_default_timezone_override_like_safe_jinja(mocker):
-    """
-    Test that mirrors exactly how safe_jinja.py uses the TimeExtension.
-    This is the simplest demonstration that environment.default_timezone works.
-    """
-    # Create environment (TimeExtension.__init__ sets default_timezone='UTC')
-    jinja2_env = ImmutableSandboxedEnvironment(extensions=[TimeExtension])
-
-    # Override the default timezone - exactly like safe_jinja.py does
-    jinja2_env.default_timezone = 'America/New_York'
-
-    # Mock arrow.now to return a fixed time
-    fixed_time = arrow.Arrow(2025, 1, 15, 12, 0, 0, tzinfo='America/New_York')
-    mock = mocker.patch("changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now", return_value=fixed_time)
-
-    # Use empty string timezone - should use the overridden default
-    template_str = "{% now '' %}"
-    output = jinja2_env.from_string(template_str).render()
-
-    # Verify arrow.now was called with the overridden timezone
-    mock.assert_called_with('America/New_York')
-    assert '2025' in output
-    assert 'Jan' in output
-
-
-def test_default_timezone_not_overridden(mocker):
-    """
-    Test that without override, the default 'UTC' from __init__ is used.
-    """
-    # Create environment (TimeExtension.__init__ sets default_timezone='UTC')
-    jinja2_env = ImmutableSandboxedEnvironment(extensions=[TimeExtension])
-
-    # DON'T override - should use 'UTC' default
-
-    # Mock arrow.now
-    fixed_time = arrow.Arrow(2025, 1, 15, 17, 0, 0, tzinfo='UTC')
-    mock = mocker.patch("changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now", return_value=fixed_time)
-
-    # Use empty string timezone - should use 'UTC' default
-    template_str = "{% now '' %}"
-    output = jinja2_env.from_string(template_str).render()
-
-    # Verify arrow.now was called with 'UTC'
-    mock.assert_called_with('UTC')
-    assert '2025' in output
-
-
-def test_datetime_format_override_like_safe_jinja(mocker):
-    """
-    Test that environment.datetime_format can be overridden after creation.
-    """
-    # Create environment (default format is '%a, %d %b %Y %H:%M:%S')
-    jinja2_env = ImmutableSandboxedEnvironment(extensions=[TimeExtension])
-
-    # Override the datetime format
-    jinja2_env.datetime_format = '%Y-%m-%d %H:%M:%S'
-
-    # Mock arrow.now
-    fixed_time = arrow.Arrow(2025, 1, 15, 14, 30, 45, tzinfo='UTC')
-    mocker.patch("changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now", return_value=fixed_time)
-
-    # Don't specify format - should use overridden default
-    template_str = "{% now 'UTC' %}"
-    output = jinja2_env.from_string(template_str).render()
-
-    # Should use custom format YYYY-MM-DD HH:MM:SS
-    assert output == '2025-01-15 14:30:45'
-
-
-def test_offset_with_overridden_timezone(mocker):
-    """
-    Test that offset operations also respect the overridden default_timezone.
-    """
-    jinja2_env = ImmutableSandboxedEnvironment(extensions=[TimeExtension])
-
-    # Override to use Europe/London
-    jinja2_env.default_timezone = 'Europe/London'
-
-    fixed_time = arrow.Arrow(2025, 1, 15, 10, 0, 0, tzinfo='Europe/London')
-    mock = mocker.patch("changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now", return_value=fixed_time)
-
-    # Use offset with empty timezone string
-    template_str = "{% now '' + 'hours=2', '%Y-%m-%d %H:%M:%S' %}"
-    output = jinja2_env.from_string(template_str).render()
-
-    # Should have called arrow.now with Europe/London
-    mock.assert_called_with('Europe/London')
-    # Should be 10:00 + 2 hours = 12:00
-    assert output == '2025-01-15 12:00:00'
-
-
-def test_weekday_parameter_converted_to_int(mocker):
-    """
-    Test that weekday parameter is properly converted from float to int.
-    This is important because arrow.shift() requires weekday as int, not float.
-    """
-    jinja2_env = ImmutableSandboxedEnvironment(extensions=[TimeExtension])
-
-    # Wednesday, Jan 15, 2025
-    fixed_time = arrow.Arrow(2025, 1, 15, 12, 0, 0, tzinfo='UTC')
-    mocker.patch("changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now", return_value=fixed_time)
-
-    # Add offset to next Monday (weekday=0)
-    template_str = "{% now 'UTC' + 'weekday=0', '%A' %}"
-    output = jinja2_env.from_string(template_str).render()
-
-    # Should be Monday
-    assert output == 'Monday'
-
-
-def test_multiple_offset_parameters(mocker):
-    """
-    Test that multiple offset parameters can be combined in one expression.
-    """
-    jinja2_env = ImmutableSandboxedEnvironment(extensions=[TimeExtension])
-
-    fixed_time = arrow.Arrow(2025, 1, 15, 10, 30, 45, tzinfo='UTC')
-    mocker.patch("changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now", return_value=fixed_time)
-
-    # Test multiple parameters: days, hours, minutes, seconds
-    template_str = "{% now 'UTC' + 'days=1,hours=2,minutes=15,seconds=10', '%Y-%m-%d %H:%M:%S' %}"
-    output = jinja2_env.from_string(template_str).render()
-
-    # 2025-01-15 10:30:45 + 1 day + 2 hours + 15 minutes + 10 seconds
-    # = 2025-01-16 12:45:55
-    assert output == '2025-01-16 12:45:55'

changedetectionio/tests/unit/test_time_handler.py

@@ -1,429 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-Comprehensive tests for time_handler module refactored to use arrow.
-
-Run from project root:
-python3 -m pytest changedetectionio/tests/unit/test_time_handler.py -v
-"""
-
-import unittest
-import arrow
-from changedetectionio import time_handler
-
-
-class TestAmIInsideTime(unittest.TestCase):
-    """Tests for the am_i_inside_time function."""
-
-    def test_current_time_within_schedule(self):
-        """Test that current time is detected as within schedule."""
-        # Get current time in a specific timezone
-        timezone_str = 'Europe/Berlin'
-        now = arrow.now(timezone_str)
-        day_of_week = now.format('dddd')
-        time_str = now.format('HH:00')  # Current hour, 0 minutes
-        duration = 60  # 60 minutes
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        self.assertTrue(result, f"Current time should be within {duration} minute window starting at {time_str}")
-
-    def test_current_time_outside_schedule(self):
-        """Test that time in the past is not within current schedule."""
-        timezone_str = 'Europe/Berlin'
-        # Get yesterday's date
-        yesterday = arrow.now(timezone_str).shift(days=-1)
-        day_of_week = yesterday.format('dddd')
-        time_str = yesterday.format('HH:mm')
-        duration = 30  # Only 30 minutes
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        self.assertFalse(result, "Yesterday's time should not be within current schedule")
-
-    def test_timezone_pacific_within_schedule(self):
-        """Test with US/Pacific timezone."""
-        timezone_str = 'US/Pacific'
-        now = arrow.now(timezone_str)
-        day_of_week = now.format('dddd')
-        time_str = now.format('HH:00')
-        duration = 120  # 2 hours
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        self.assertTrue(result)
-
-    def test_timezone_tokyo_within_schedule(self):
-        """Test with Asia/Tokyo timezone."""
-        timezone_str = 'Asia/Tokyo'
-        now = arrow.now(timezone_str)
-        day_of_week = now.format('dddd')
-        time_str = now.format('HH:00')
-        duration = 90  # 1.5 hours
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        self.assertTrue(result)
-
-    def test_schedule_crossing_midnight(self):
-        """Test schedule that crosses midnight."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-
-        # Set schedule to start 23:30 with 120 minute duration (crosses midnight)
-        day_of_week = now.format('dddd')
-        time_str = "23:30"
-        duration = 120  # 2 hours - goes into next day
-
-        # If we're at 00:15 the next day, we should still be in the schedule
-        if now.hour == 0 and now.minute < 30:
-            # We're in the time window that spilled over from yesterday
-            result = time_handler.am_i_inside_time(
-                day_of_week=day_of_week,
-                time_str=time_str,
-                timezone_str=timezone_str,
-                duration=duration
-            )
-            # This might be true or false depending on exact time
-            self.assertIsInstance(result, bool)
-
-    def test_invalid_day_of_week(self):
-        """Test that invalid day raises ValueError."""
-        with self.assertRaises(ValueError) as context:
-            time_handler.am_i_inside_time(
-                day_of_week="Funday",
-                time_str="12:00",
-                timezone_str="UTC",
-                duration=60
-            )
-        self.assertIn("Invalid day_of_week", str(context.exception))
-
-    def test_invalid_time_format(self):
-        """Test that invalid time format raises ValueError."""
-        with self.assertRaises(ValueError) as context:
-            time_handler.am_i_inside_time(
-                day_of_week="Monday",
-                time_str="25:99",
-                timezone_str="UTC",
-                duration=60
-            )
-        self.assertIn("Invalid time_str", str(context.exception))
-
-    def test_invalid_time_format_non_numeric(self):
-        """Test that non-numeric time raises ValueError."""
-        with self.assertRaises(ValueError) as context:
-            time_handler.am_i_inside_time(
-                day_of_week="Monday",
-                time_str="twelve:thirty",
-                timezone_str="UTC",
-                duration=60
-            )
-        self.assertIn("Invalid time_str", str(context.exception))
-
-    def test_invalid_timezone(self):
-        """Test that invalid timezone raises ValueError."""
-        with self.assertRaises(ValueError) as context:
-            time_handler.am_i_inside_time(
-                day_of_week="Monday",
-                time_str="12:00",
-                timezone_str="Invalid/Timezone",
-                duration=60
-            )
-        self.assertIn("Invalid timezone_str", str(context.exception))
-
-    def test_short_duration(self):
-        """Test with very short duration (15 minutes default)."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        day_of_week = now.format('dddd')
-        time_str = now.format('HH:mm')
-        duration = 15  # Default duration
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        self.assertTrue(result, "Current time should be within 15 minute window")
-
-    def test_long_duration(self):
-        """Test with long duration (24 hours)."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        day_of_week = now.format('dddd')
-        # Set time to current hour
-        time_str = now.format('HH:00')
-        duration = 1440  # 24 hours in minutes
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        self.assertTrue(result, "Current time should be within 24 hour window")
-
-    def test_case_insensitive_day(self):
-        """Test that day of week is case insensitive."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        day_of_week = now.format('dddd').lower()  # lowercase day
-        time_str = now.format('HH:00')
-        duration = 60
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        self.assertTrue(result, "Lowercase day should work")
-
-    def test_edge_case_midnight(self):
-        """Test edge case at exactly midnight."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        day_of_week = now.format('dddd')
-        time_str = "00:00"
-        duration = 60
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        # Should be true if we're in the first hour of the day
-        if now.hour == 0:
-            self.assertTrue(result)
-
-    def test_edge_case_end_of_day(self):
-        """Test edge case near end of day."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        day_of_week = now.format('dddd')
-        time_str = "23:45"
-        duration = 30  # 30 minutes crosses midnight
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        # Result depends on current time
-        self.assertIsInstance(result, bool)
-
-
-class TestIsWithinSchedule(unittest.TestCase):
-    """Tests for the is_within_schedule function."""
-
-    def test_schedule_disabled(self):
-        """Test that disabled schedule returns False."""
-        time_schedule_limit = {'enabled': False}
-        result = time_handler.is_within_schedule(time_schedule_limit)
-        self.assertFalse(result)
-
-    def test_schedule_none(self):
-        """Test that None schedule returns False."""
-        result = time_handler.is_within_schedule(None)
-        self.assertFalse(result)
-
-    def test_schedule_empty_dict(self):
-        """Test that empty dict returns False."""
-        result = time_handler.is_within_schedule({})
-        self.assertFalse(result)
-
-    def test_schedule_enabled_but_day_disabled(self):
-        """Test schedule enabled but current day disabled."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        current_day = now.format('dddd').lower()
-
-        time_schedule_limit = {
-            'enabled': True,
-            'timezone': timezone_str,
-            current_day: {
-                'enabled': False,
-                'start_time': '09:00',
-                'duration': {'hours': 8, 'minutes': 0}
-            }
-        }
-
-        result = time_handler.is_within_schedule(time_schedule_limit)
-        self.assertFalse(result, "Disabled day should return False")
-
-    def test_schedule_enabled_within_time(self):
-        """Test schedule enabled and within time window."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        current_day = now.format('dddd').lower()
-        current_hour = now.format('HH:00')
-
-        time_schedule_limit = {
-            'enabled': True,
-            'timezone': timezone_str,
-            current_day: {
-                'enabled': True,
-                'start_time': current_hour,
-                'duration': {'hours': 2, 'minutes': 0}
-            }
-        }
-
-        result = time_handler.is_within_schedule(time_schedule_limit)
-        self.assertTrue(result, "Current time should be within schedule")
-
-    def test_schedule_enabled_outside_time(self):
-        """Test schedule enabled but outside time window."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        current_day = now.format('dddd').lower()
-        # Set time to 3 hours ago
-        past_time = now.shift(hours=-3).format('HH:mm')
-
-        time_schedule_limit = {
-            'enabled': True,
-            'timezone': timezone_str,
-            current_day: {
-                'enabled': True,
-                'start_time': past_time,
-                'duration': {'hours': 1, 'minutes': 0}  # Only 1 hour duration
-            }
-        }
-
-        result = time_handler.is_within_schedule(time_schedule_limit)
-        self.assertFalse(result, "3 hours ago with 1 hour duration should be False")
-
-    def test_schedule_with_default_timezone(self):
-        """Test schedule without timezone uses default."""
-        now = arrow.now('America/New_York')
-        current_day = now.format('dddd').lower()
-        current_hour = now.format('HH:00')
-
-        time_schedule_limit = {
-            'enabled': True,
-            # No timezone specified
-            current_day: {
-                'enabled': True,
-                'start_time': current_hour,
-                'duration': {'hours': 2, 'minutes': 0}
-            }
-        }
-
-        # Should use default UTC, but since we're testing with NY time,
-        # the result depends on time difference
-        result = time_handler.is_within_schedule(
-            time_schedule_limit,
-            default_tz='America/New_York'
-        )
-        self.assertTrue(result, "Should work with default timezone")
-
-    def test_schedule_different_timezones(self):
-        """Test schedule works correctly across different timezones."""
-        # Test with Tokyo timezone
-        timezone_str = 'Asia/Tokyo'
-        now = arrow.now(timezone_str)
-        current_day = now.format('dddd').lower()
-        current_hour = now.format('HH:00')
-
-        time_schedule_limit = {
-            'enabled': True,
-            'timezone': timezone_str,
-            current_day: {
-                'enabled': True,
-                'start_time': current_hour,
-                'duration': {'hours': 1, 'minutes': 30}
-            }
-        }
-
-        result = time_handler.is_within_schedule(time_schedule_limit)
-        self.assertTrue(result)
-
-    def test_schedule_with_minutes_in_duration(self):
-        """Test schedule with minutes specified in duration."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        current_day = now.format('dddd').lower()
-        current_time = now.format('HH:mm')
-
-        time_schedule_limit = {
-            'enabled': True,
-            'timezone': timezone_str,
-            current_day: {
-                'enabled': True,
-                'start_time': current_time,
-                'duration': {'hours': 0, 'minutes': 45}
-            }
-        }
-
-        result = time_handler.is_within_schedule(time_schedule_limit)
-        self.assertTrue(result, "Should handle minutes in duration")
-
-    def test_schedule_with_timezone_whitespace(self):
-        """Test that timezone with whitespace is handled."""
-        timezone_str = '  UTC  '
-        now = arrow.now('UTC')
-        current_day = now.format('dddd').lower()
-        current_hour = now.format('HH:00')
-
-        time_schedule_limit = {
-            'enabled': True,
-            'timezone': timezone_str,
-            current_day: {
-                'enabled': True,
-                'start_time': current_hour,
-                'duration': {'hours': 1, 'minutes': 0}
-            }
-        }
-
-        result = time_handler.is_within_schedule(time_schedule_limit)
-        self.assertTrue(result, "Should handle timezone with whitespace")
-
-
-class TestWeekdayEnum(unittest.TestCase):
-    """Tests for the Weekday enum."""
-
-    def test_weekday_values(self):
-        """Test that weekday enum has correct values."""
-        self.assertEqual(time_handler.Weekday.Monday, 0)
-        self.assertEqual(time_handler.Weekday.Tuesday, 1)
-        self.assertEqual(time_handler.Weekday.Wednesday, 2)
-        self.assertEqual(time_handler.Weekday.Thursday, 3)
-        self.assertEqual(time_handler.Weekday.Friday, 4)
-        self.assertEqual(time_handler.Weekday.Saturday, 5)
-        self.assertEqual(time_handler.Weekday.Sunday, 6)
-
-    def test_weekday_string_access(self):
-        """Test accessing weekday enum by string."""
-        self.assertEqual(time_handler.Weekday['Monday'], 0)
-        self.assertEqual(time_handler.Weekday['Sunday'], 6)
-
-
-if __name__ == '__main__':
-    unittest.main()

changedetectionio/tests/util.py

@@ -188,10 +188,6 @@ def new_live_server_setup(live_server):
         ctype = request.args.get('content_type')
         status_code = request.args.get('status_code')
         content = request.args.get('content') or None
-        delay = int(request.args.get('delay', 0))
-
-        if delay:
-            time.sleep(delay)
 
         # Used to just try to break the header detection
         uppercase_headers = request.args.get('uppercase_headers')

changedetectionio/time_handler.py

@@ -1,5 +1,6 @@
-import arrow
+from datetime import timedelta, datetime
 from enum import IntEnum
+from zoneinfo import ZoneInfo
 
 
 class Weekday(IntEnum):
@@ -39,65 +40,54 @@ def am_i_inside_time(
 
     # Parse the start time
     try:
-        hour, minute = map(int, time_str.split(':'))
-        if not (0 <= hour <= 23 and 0 <= minute <= 59):
-            raise ValueError
-    except (ValueError, AttributeError):
+        target_time = datetime.strptime(time_str, '%H:%M').time()
+    except ValueError:
         raise ValueError(f"Invalid time_str: '{time_str}'. Must be in 'HH:MM' format.")
 
-    # Get the current time in the specified timezone
+    # Define the timezone
     try:
-        now_tz = arrow.now(timezone_str.strip())
-    except Exception as e:
+        tz = ZoneInfo(timezone_str)
+    except Exception:
         raise ValueError(f"Invalid timezone_str: '{timezone_str}'. Must be a valid timezone identifier.")
 
+    # Get the current time in the specified timezone
+    now_tz = datetime.now(tz)
+
     # Check if the current day matches the target day or overlaps due to duration
     current_weekday = now_tz.weekday()
-    # Create start datetime for today in target timezone
-    start_datetime_tz = now_tz.replace(hour=hour, minute=minute, second=0, microsecond=0)
+    start_datetime_tz = datetime.combine(now_tz.date(), target_time, tzinfo=tz)
 
     # Handle previous day's overlap
     if target_weekday == (current_weekday - 1) % 7:
         # Calculate start and end times for the overlap from the previous day
-        start_datetime_tz = start_datetime_tz.shift(days=-1)
-        end_datetime_tz = start_datetime_tz.shift(minutes=duration)
+        start_datetime_tz -= timedelta(days=1)
+        end_datetime_tz = start_datetime_tz + timedelta(minutes=duration)
         if start_datetime_tz <= now_tz < end_datetime_tz:
             return True
 
     # Handle current day's range
     if target_weekday == current_weekday:
-        end_datetime_tz = start_datetime_tz.shift(minutes=duration)
+        end_datetime_tz = start_datetime_tz + timedelta(minutes=duration)
         if start_datetime_tz <= now_tz < end_datetime_tz:
             return True
 
     # Handle next day's overlap
     if target_weekday == (current_weekday + 1) % 7:
-        end_datetime_tz = start_datetime_tz.shift(minutes=duration)
-        if now_tz < start_datetime_tz and now_tz.shift(days=1) < end_datetime_tz:
+        end_datetime_tz = start_datetime_tz + timedelta(minutes=duration)
+        if now_tz < start_datetime_tz and now_tz + timedelta(days=1) < end_datetime_tz:
             return True
 
     return False
 
 
 def is_within_schedule(time_schedule_limit, default_tz="UTC"):
-    """
-    Check if the current time is within a scheduled time window.
-
-    Parameters:
-        time_schedule_limit (dict): Schedule configuration with timezone, day settings, etc.
-        default_tz (str): Default timezone to use if not specified. Default is 'UTC'.
-
-    Returns:
-        bool: True if current time is within the schedule, False otherwise.
-    """
     if time_schedule_limit and time_schedule_limit.get('enabled'):
         # Get the timezone the time schedule is in, so we know what day it is there
         tz_name = time_schedule_limit.get('timezone')
         if not tz_name:
             tz_name = default_tz
 
-        # Get current day name in the target timezone
-        now_day_name_in_tz = arrow.now(tz_name.strip()).format('dddd')
+        now_day_name_in_tz = datetime.now(ZoneInfo(tz_name.strip())).strftime('%A')
         selected_day_schedule = time_schedule_limit.get(now_day_name_in_tz.lower())
         if not selected_day_schedule.get('enabled'):
             return False

requirements.txt

@@ -42,9 +42,6 @@ jsonpath-ng~=1.5.3
 # Notification library
 apprise==1.9.5
 
-# Lightweight URL linkifier for notifications
-linkify-it-py
-
 # - Needed for apprise/spush, and maybe others? hopefully doesnt trigger a rust compile.
 # - Requires extra wheel for rPi, adds build time for arm/v8 which is not in piwheels
 # Pinned to 43.0.1 for ARM compatibility (45.x may not have pre-built ARM wheels)
@@ -76,7 +73,7 @@ werkzeug==3.0.6
 
 # Templating, so far just in the URLs but in the future can be for the notifications also
 jinja2~=3.1
-arrow
+jinja2-time
 openpyxl
 # https://peps.python.org/pep-0508/#environment-markers
 # https://github.com/dgtlmoon/changedetection.io/pull/1009
@@ -92,7 +89,6 @@ pyppeteerstealth>=0.0.4
 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup
 pytest ~=7.2
 pytest-flask ~=1.2
-pytest-mock ~=3.15
 
 # Anything 4.0 and up but not 5.0
 jsonschema ~= 4.0

setup.py

@@ -5,8 +5,6 @@ import re
 import sys
 
 from setuptools import setup, find_packages
-from setuptools.command.build_py import build_py
-import shutil
 
 here = os.path.abspath(os.path.dirname(__file__))
 
@@ -24,20 +22,6 @@ def find_version(*file_paths):
     raise RuntimeError("Unable to find version string.")
 
 
-class BuildPyCommand(build_py):
-    """Custom build command to copy api-spec.yaml to the package."""
-    def run(self):
-        build_py.run(self)
-        # Ensure the docs directory exists in the build output
-        docs_dir = os.path.join(self.build_lib, 'changedetectionio', 'docs')
-        os.makedirs(docs_dir, exist_ok=True)
-        # Copy api-spec.yaml to the package
-        shutil.copy(
-            os.path.join(here, 'docs', 'api-spec.yaml'),
-            os.path.join(docs_dir, 'api-spec.yaml')
-        )
-
-
 install_requires = open('requirements.txt').readlines()
 
 setup(
@@ -53,10 +37,9 @@ setup(
     scripts=["changedetection.py"],
     author='dgtlmoon',
     url='https://changedetection.io',
-    packages=find_packages(include=['changedetectionio', 'changedetectionio.*']),
+    packages=['changedetectionio'],
     include_package_data=True,
     install_requires=install_requires,
-    cmdclass={'build_py': BuildPyCommand},
     license="Apache License 2.0",
     python_requires=">= 3.10",
     classifiers=['Intended Audience :: Customer Service',