Ensure JSON is always nicely formatted

.github/workflows/pypi-release.yml

@@ -28,7 +28,7 @@ jobs:
 
 
   test-pypi-package:
-    name: Test the built package works basically.
+    name: Test the built 📦 package works basically.
     runs-on: ubuntu-latest
     needs:
     - build
@@ -42,39 +42,18 @@ jobs:
       uses: actions/setup-python@v6
       with:
         python-version: '3.11'
-
     - name: Test that the basic pip built package runs without error
       run: |
         set -ex
         ls -alR 
         
-        # Install the first wheel found in dist/
-        WHEEL=$(find dist -type f -name "*.whl" -print -quit)
-        echo Installing $WHEEL
-        python3 -m pip install --upgrade pip
-        python3 -m pip install "$WHEEL"
+        # Find and install the first .whl file
+        find dist -type f -name "*.whl" -exec pip3 install {} \; -quit
         changedetection.io -d /tmp -p 10000 &
         
         sleep 3
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/ >/dev/null
-        
-        # --- API test ---
-        # This also means that the docs/api-spec.yml was shipped and could be read
-        test -f /tmp/url-watches.json
-        API_KEY=$(jq -r '.. | .api_access_token? // empty' /tmp/url-watches.json)
-        echo Test API KEY is $API_KEY
-        curl -X POST "http://127.0.0.1:10000/api/v1/watch" \
-          -H "x-api-key: ${API_KEY}" \
-          -H "Content-Type: application/json" \
-          --show-error --fail \
-          --retry 6 --retry-delay 1 --retry-connrefused \
-          -d '{
-            "url": "https://example.com",
-            "title": "Example Site Monitor",
-            "time_between_check": { "hours": 1 }
-          }'
-          
         killall changedetection.io
 
 

.github/workflows/test-stack-reusable-workflow.yml

@@ -54,10 +54,7 @@ jobs:
 
       - name: Spin up ancillary SMTP+Echo message test server
         run: |
-          # Debug SMTP server/echo message back server, telnet 11080 to it should immediately bounce back the most recent message that tried to send (then you can see if cdio tried to send, the format, etc)
-          # 11025 is the SMTP port for testing
-          # apprise example would be 'mailto://changedetection@localhost:11025/?to=fff@home.com  (it will also echo to STDOUT)
-          # telnet localhost 11080
+          # Debug SMTP server/echo message back server
           docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'pip3 install aiosmtpd && python changedetectionio/tests/smtp/smtp-test-server.py'
           docker ps
 
@@ -256,30 +253,6 @@ jobs:
           docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout-${{ env.PYTHON_VERSION }}.txt
           docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr-${{ env.PYTHON_VERSION }}.txt
 
-      - name: Extract and display memory test report
-        if: always()
-        run: |
-          # Extract test-memory.log from the container
-          echo "Extracting test-memory.log from container..."
-          docker cp test-cdio-basic-tests:/app/changedetectionio/test-memory.log output-logs/test-memory-${{ env.PYTHON_VERSION }}.log || echo "test-memory.log not found in container"
-
-          # Display the memory log contents for immediate visibility in workflow output
-          echo "=== Top 10 Highest Peak Memory Tests ==="
-          if [ -f output-logs/test-memory-${{ env.PYTHON_VERSION }}.log ]; then
-            # Sort by peak memory value (extract number before MB and sort numerically, reverse order)
-            grep "Peak memory:" output-logs/test-memory-${{ env.PYTHON_VERSION }}.log | \
-              sed 's/.*Peak memory: //' | \
-              paste -d'|' - <(grep "Peak memory:" output-logs/test-memory-${{ env.PYTHON_VERSION }}.log) | \
-              sort -t'|' -k1 -nr | \
-              cut -d'|' -f2 | \
-              head -10
-            echo ""
-            echo "=== Full Memory Test Report ==="
-            cat output-logs/test-memory-${{ env.PYTHON_VERSION }}.log
-          else
-            echo "No memory log available"
-          fi
-
       - name: Store everything including test-datastore
         if: always()
         uses: actions/upload-artifact@v4

MANIFEST.in

@@ -1,9 +1,7 @@
 recursive-include changedetectionio/api *
-include docs/api-spec.yaml
 recursive-include changedetectionio/blueprint *
 recursive-include changedetectionio/conditions *
 recursive-include changedetectionio/content_fetchers *
-recursive-include changedetectionio/jinja2_custom *
 recursive-include changedetectionio/model *
 recursive-include changedetectionio/notification *
 recursive-include changedetectionio/processors *

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.50.29'
+__version__ = '0.50.20'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/api/__init__.py

@@ -37,10 +37,6 @@ def get_openapi_spec():
     from openapi_core import OpenAPI  # Lazy import - saves ~10.7 MB on startup
 
     spec_path = os.path.join(os.path.dirname(__file__), '../../docs/api-spec.yaml')
-    if not os.path.exists(spec_path):
-        # Possibly for pip3 packages
-        spec_path = os.path.join(os.path.dirname(__file__), '../docs/api-spec.yaml')
-
     with open(spec_path, 'r') as f:
         spec_dict = yaml.safe_load(f)
     _openapi_spec = OpenAPI.from_dict(spec_dict)

changedetectionio/async_update_worker.py

@@ -334,10 +334,6 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
                             if update_handler.fetcher.content or (not update_handler.fetcher.content and empty_pages_are_a_change):
                                 watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=int(fetch_start_time))
 
-                            # Explicitly delete large content variables to free memory IMMEDIATELY after saving
-                            # These are no longer needed after being saved to history
-                            del contents
-
                             # Send notifications on second+ check
                             if watch.history_n >= 2:
                                 logger.info(f"Change detected in UUID {uuid} - {watch['url']}")
@@ -376,12 +372,6 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
                 datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - fetch_start_time, 3),
                                                                'check_count': count})
 
-                # NOW clear fetcher content - after all processing is complete
-                # This is the last point where we need the fetcher data
-                if update_handler and hasattr(update_handler, 'fetcher') and update_handler.fetcher:
-                    update_handler.fetcher.clear_content()
-                    logger.debug(f"Cleared fetcher content for UUID {uuid}")
-
         except Exception as e:
             logger.error(f"Worker {worker_id} unexpected error processing {uuid}: {e}")
             logger.error(f"Worker {worker_id} traceback:", exc_info=True)
@@ -402,28 +392,7 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
                         #logger.info(f"Worker {worker_id} sending completion signal for UUID {watch['uuid']}")
                         watch_check_update.send(watch_uuid=watch['uuid'])
 
-                    # Explicitly clean up update_handler and all its references
-                    if update_handler:
-                        # Clear fetcher content using the proper method
-                        if hasattr(update_handler, 'fetcher') and update_handler.fetcher:
-                            update_handler.fetcher.clear_content()
-
-                        # Clear processor references
-                        if hasattr(update_handler, 'content_processor'):
-                            update_handler.content_processor = None
-
-                        update_handler = None
-
-                    # Clear local contents variable if it still exists
-                    if 'contents' in locals():
-                        del contents
-
-                    # Note: We don't set watch = None here because:
-                    # 1. watch is just a local reference to datastore.data['watching'][uuid]
-                    # 2. Setting it to None doesn't affect the datastore
-                    # 3. GC can't collect the object anyway (still referenced by datastore)
-                    # 4. It would just cause confusion
-
+                    update_handler = None
                     logger.debug(f"Worker {worker_id} completed watch {uuid} in {time.time()-fetch_start_time:.2f}s")
                 except Exception as cleanup_error:
                     logger.error(f"Worker {worker_id} error during cleanup: {cleanup_error}")

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -6,7 +6,7 @@ from loguru import logger
 
 from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT
 from changedetectionio.content_fetchers.base import manage_user_agent
-from changedetectionio.jinja2_custom import render as jinja_render
+from changedetectionio.safe_jinja import render as jinja_render
 
 
 

changedetectionio/blueprint/check_proxies/__init__.py

@@ -33,7 +33,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def long_task(uuid, preferred_proxy):
         import time
         from changedetectionio.content_fetchers import exceptions as content_fetcher_exceptions
-        from changedetectionio.jinja2_custom import render as jinja_render
+        from changedetectionio.safe_jinja import render as jinja_render
 
         status = {'status': '', 'length': 0, 'text': ''}
 

changedetectionio/blueprint/rss/blueprint.py

@@ -1,6 +1,5 @@
 
-from changedetectionio.jinja2_custom import render as jinja_render
-from changedetectionio.notification.handler import apply_service_tweaks
+from changedetectionio.safe_jinja import render as jinja_render
 from changedetectionio.store import ChangeDetectionStore
 from feedgen.feed import FeedGenerator
 from flask import Blueprint, make_response, request, url_for, redirect
@@ -121,13 +120,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                     html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]),
                                                  newest_version_file_contents=watch.get_history_snapshot(dates[-1]),
                                                  include_equal=False,
-                                                 line_feed_sep="<br>"
+                                                 line_feed_sep="<br>",
+                                                 html_colour=html_colour_enable
                                                  )
-
-
-                    requested_output_format = 'htmlcolor' if html_colour_enable else 'html'
-                    html_diff = apply_service_tweaks(url='', n_body=html_diff, n_title=None, requested_output_format=requested_output_format)
-
                 except FileNotFoundError as e:
                     html_diff = f"History snapshot file for watch {watch.get('uuid')}@{watch.last_changed} - '{watch.get('title')} not found."
 

changedetectionio/blueprint/settings/__init__.py

@@ -119,7 +119,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                                 hide_remove_pass=os.getenv("SALTED_PASS", False),
                                 min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)),
                                 settings_application=datastore.data['settings']['application'],
-                                timezone_default_config=datastore.data['settings']['application'].get('scheduler_timezone_default'),
+                                timezone_default_config=datastore.data['settings']['application'].get('timezone'),
                                 utc_time=utc_time,
                                 )
 

changedetectionio/blueprint/settings/templates/settings.html

@@ -1,7 +1,7 @@
 {% extends 'base.html' %}
 
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, render_ternary_field, render_fieldlist_with_inline_errors %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, render_ternary_field %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script>
     const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="global-settings")}}";
@@ -72,23 +72,25 @@
                         <span class="pure-form-message-inline">Allow access to view watch diff page when password is enabled (Good for sharing the diff page)
                         </span>
                     </div>
+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }}
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.application.form.rss_content_format) }}
+                        <span class="pure-form-message-inline">Love RSS? Does your reader support HTML? Set it here</span>
+                    </div>
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}
                         <span class="pure-form-message-inline">When a request returns no content, or the HTML does not contain any text, is this considered a change?</span>
                     </div>
-                    <div class="grey-form-border">
-                        <div class="pure-control-group">
-                            {{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }}
-                        </div>
-                        <div class="pure-control-group">
-                            {{ render_field(form.application.form.rss_content_format) }}
-                            <span class="pure-form-message-inline">Love RSS? Does your reader support HTML? Set it here</span>
-                        </div>
-                        <div class="pure-control-group">
-                            {{ render_checkbox_field(form.application.form.rss_reader_mode) }}
-                            <span class="pure-form-message-inline">Transforms RSS/RDF feed watches into beautiful text only</span>
-                        </div>
+                {% if form.requests.proxy %}
+                    <div class="pure-control-group inline-radio">
+                        {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
+                        <span class="pure-form-message-inline">
+                        Choose a default proxy for all watches
+                        </span>
                     </div>
+                {% endif %}
                 </fieldset>
             </div>
 
@@ -131,10 +133,6 @@
                     <span class="pure-form-message-inline">Number of concurrent workers to process watches. More workers = faster processing but higher memory usage.<br>
                     Currently running: <strong>{{ worker_info.count }}</strong> operational {{ worker_info.type }} workers{% if worker_info.active_workers > 0 %} ({{ worker_info.active_workers }} actively processing){% endif %}.</span>
                 </div>
-                <div class="pure-control-group">
-                    {{ render_field(form.requests.form.timeout) }}
-                    <span class="pure-form-message-inline">For regular plain requests (not chrome based), maximum number of seconds until timeout, 1-999.<br>
-                </div>
                 <div class="pure-control-group inline-radio">
                     {{ render_field(form.requests.form.default_ua) }}
                     <span class="pure-form-message-inline">
@@ -238,7 +236,7 @@ nav
                     <p><strong>UTC Time &amp Date from Server:</strong> <span id="utc-time" >{{ utc_time }}</span></p>
                     <p><strong>Local Time &amp Date in Browser:</strong> <span class="local-time" data-utc="{{ utc_time }}"></span></p>
                     <p>
-                       {{ render_field(form.application.form.scheduler_timezone_default) }}
+                       {{ render_field(form.application.form.timezone) }}
                         <datalist id="timezones" style="display: none;">
                             {% for tz_name in available_timezones %}
                                 <option value="{{ tz_name }}">{{ tz_name }}</option>
@@ -316,27 +314,17 @@ nav
                <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
 
                 <div class="pure-control-group" id="extra-proxies-setting">
-                {{ render_fieldlist_with_inline_errors(form.requests.form.extra_proxies) }}
+                {{ render_field(form.requests.form.extra_proxies) }}
                 <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span><br>
                 <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
-                {% if form.requests.proxy %}
-                <div>
-                <br>
-                    <div class="inline-radio">
-                        {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
-                        <span class="pure-form-message-inline">Choose a default proxy for all watches</span>
-                    </div>
-                </div>
-                {% endif %}
                 </div>
                 <div class="pure-control-group" id="extra-browsers-setting">
                     <p>
                     <span class="pure-form-message-inline"><i>Extra Browsers</i> can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.</span><br>
                     <span class="pure-form-message-inline">Simply paste the connection address into the box, <a href="https://changedetection.io/tutorial/using-bright-datas-scraping-browser-pass-captchas-and-other-protection-when-monitoring">More instructions and examples here</a> </span>
                     </p>
-                    {{ render_fieldlist_with_inline_errors(form.requests.form.extra_browsers) }}
+                    {{ render_field(form.requests.form.extra_browsers) }}
                 </div>
-            
             </div>
             <div id="actions">
                 <div class="pure-control-group">

changedetectionio/blueprint/ui/edit.py

@@ -187,7 +187,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
             tz_name = time_schedule_limit.get('timezone')
             if not tz_name:
-                tz_name = datastore.data['settings']['application'].get('scheduler_timezone_default', os.getenv('TZ', 'UTC').strip())
+                tz_name = datastore.data['settings']['application'].get('timezone', 'UTC')
 
             if time_schedule_limit and time_schedule_limit.get('enabled'):
                 try:
@@ -257,7 +257,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                 'system_has_webdriver_configured': os.getenv('WEBDRIVER_URL'),
                 'ui_edit_stats_extras': collect_ui_edit_stats_extras(watch),
                 'visual_selector_data_ready': datastore.visualselector_data_is_ready(watch_uuid=uuid),
-                'timezone_default_config': datastore.data['settings']['application'].get('scheduler_timezone_default'),
+                'timezone_default_config': datastore.data['settings']['application'].get('timezone'),
                 'using_global_webdriver_wait': not default['webdriver_delay'],
                 'uuid': uuid,
                 'watch': watch,

changedetectionio/blueprint/ui/notification.py

@@ -2,7 +2,6 @@ from flask import Blueprint, request, make_response
 import random
 from loguru import logger
 
-from changedetectionio.notification_service import NotificationContextData
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.auth_decorator import login_optionally_required
 
@@ -20,7 +19,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         import apprise
         from changedetectionio.notification.handler import process_notification
         from changedetectionio.notification.apprise_plugin.assets import apprise_asset
-        from changedetectionio.jinja2_custom import render as jinja_render
 
         from changedetectionio.notification.apprise_plugin.custom_handlers import apprise_http_custom_handler
 
@@ -63,20 +61,16 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             return 'Error: No Notification URLs set/found'
 
         for n_url in notification_urls:
-            # We are ONLY validating the apprise:// part here, convert all tags to something so as not to break apprise URLs
-            generic_notification_context_data = NotificationContextData()
-            generic_notification_context_data.set_random_for_validation()
-            n_url = jinja_render(template_str=n_url, **generic_notification_context_data).strip()
             if len(n_url.strip()):
                 if not apobj.add(n_url):
                     return f'Error:  {n_url} is not a valid AppRise URL.'
 
         try:
             # use the same as when it is triggered, but then override it with the form test values
-            n_object = NotificationContextData({
+            n_object = {
                 'watch_url': request.form.get('window_url', "https://changedetection.io"),
                 'notification_urls': notification_urls
-            })
+            }
 
             # Only use if present, if not set in n_object it should use the default system value
             if 'notification_format' in request.form and request.form['notification_format'].strip():

changedetectionio/content_fetchers/base.py

@@ -64,19 +64,6 @@ class Fetcher():
     # Time ONTOP of the system defined env minimum time
     render_extract_delay = 0
 
-    def clear_content(self):
-        """
-        Explicitly clear all content from memory to free up heap space.
-        Call this after content has been saved to disk.
-        """
-        self.content = None
-        if hasattr(self, 'raw_content'):
-            self.raw_content = None
-        self.screenshot = None
-        self.xpath_data = None
-        # Keep headers and status_code as they're small
-        logger.trace("Fetcher content cleared from memory")
-
     @abstractmethod
     def get_error(self):
         return self.error
@@ -141,7 +128,7 @@ class Fetcher():
     async def iterate_browser_steps(self, start_url=None):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
         from playwright._impl._errors import TimeoutError, Error
-        from changedetectionio.jinja2_custom import render as jinja_render
+        from changedetectionio.safe_jinja import render as jinja_render
         step_n = 0
 
         if self.browser_steps is not None and len(self.browser_steps):

changedetectionio/content_fetchers/requests.py

@@ -51,7 +51,6 @@ class fetcher(Fetcher):
 
         session = requests.Session()
 
-
         if strtobool(os.getenv('ALLOW_FILE_URI', 'false')) and url.startswith('file://'):
             from requests_file import FileAdapter
             session.mount('file://', FileAdapter())

changedetectionio/diff.py

@@ -1,21 +1,8 @@
 import difflib
 from typing import List, Iterator, Union
 
-HTML_REMOVED_STYLE = "background-color: #fadad7; color: #b30000;"
-HTML_ADDED_STYLE = "background-color: #eaf2c2; color: #406619;"
-
-# These get set to html or telegram type or discord compatible or whatever in handler.py
-REMOVED_PLACEMARKER_OPEN = '<<<removed_PLACEMARKER_OPEN'
-REMOVED_PLACEMARKER_CLOSED = '<<<removed_PLACEMARKER_CLOSED'
-
-ADDED_PLACEMARKER_OPEN = '<<<added_PLACEMARKER_OPEN'
-ADDED_PLACEMARKER_CLOSED = '<<<added_PLACEMARKER_CLOSED'
-
-CHANGED_PLACEMARKER_OPEN = '<<<changed_PLACEMARKER_OPEN'
-CHANGED_PLACEMARKER_CLOSED = '<<<changed_PLACEMARKER_CLOSED'
-
-CHANGED_INTO_PLACEMARKER_OPEN = '<<<changed_into_PLACEMARKER_OPEN'
-CHANGED_INTO_PLACEMARKER_CLOSED = '<<<changed_into_PLACEMARKER_CLOSED'
+REMOVED_STYLE = "background-color: #fadad7; color: #b30000;"
+ADDED_STYLE = "background-color: #eaf2c2; color: #406619;"
 
 def same_slicer(lst: List[str], start: int, end: int) -> List[str]:
     """Return a slice of the list, or a single element if start == end."""
@@ -28,7 +15,8 @@ def customSequenceMatcher(
     include_removed: bool = True,
     include_added: bool = True,
     include_replaced: bool = True,
-    include_change_type_prefix: bool = True
+    include_change_type_prefix: bool = True,
+    html_colour: bool = False
 ) -> Iterator[List[str]]:
     """
     Compare two sequences and yield differences based on specified parameters.
@@ -41,6 +29,8 @@ def customSequenceMatcher(
         include_added (bool): Include added parts
         include_replaced (bool): Include replaced parts
         include_change_type_prefix (bool): Add prefixes to indicate change types
+        html_colour (bool): Use HTML background colors for differences
+
     Yields:
         List[str]: Differences between sequences
     """
@@ -52,22 +42,22 @@ def customSequenceMatcher(
         if include_equal and tag == 'equal':
             yield before[alo:ahi]
         elif include_removed and tag == 'delete':
-            if include_change_type_prefix:
-                yield [f'{REMOVED_PLACEMARKER_OPEN}{line}{REMOVED_PLACEMARKER_CLOSED}' for line in same_slicer(before, alo, ahi)]
+            if html_colour:
+                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)]
             else:
-                yield same_slicer(before, alo, ahi)
+                yield [f"(removed) {line}" for line in same_slicer(before, alo, ahi)] if include_change_type_prefix else same_slicer(before, alo, ahi)
         elif include_replaced and tag == 'replace':
-            if include_change_type_prefix:
-                yield [f'{CHANGED_PLACEMARKER_OPEN}{line}{CHANGED_PLACEMARKER_CLOSED}' for line in same_slicer(before, alo, ahi)] + \
-                      [f'{CHANGED_INTO_PLACEMARKER_OPEN}{line}{CHANGED_INTO_PLACEMARKER_CLOSED}' for line in same_slicer(after, blo, bhi)]
+            if html_colour:
+                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)] + \
+                      [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
             else:
-                yield same_slicer(before, alo, ahi) + same_slicer(after, blo, bhi)
+                yield [f"(changed) {line}" for line in same_slicer(before, alo, ahi)] + \
+                      [f"(into) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(before, alo, ahi) + same_slicer(after, blo, bhi)
         elif include_added and tag == 'insert':
-            if include_change_type_prefix:
-                yield [f'{ADDED_PLACEMARKER_OPEN}{line}{ADDED_PLACEMARKER_CLOSED}' for line in same_slicer(after, blo, bhi)]
+            if html_colour:
+                yield [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
             else:
-                yield same_slicer(after, blo, bhi)
-
+                yield [f"(added) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(after, blo, bhi)
 
 def render_diff(
     previous_version_file_contents: str,
@@ -78,7 +68,8 @@ def render_diff(
     include_replaced: bool = True,
     line_feed_sep: str = "\n",
     include_change_type_prefix: bool = True,
-    patch_format: bool = False
+    patch_format: bool = False,
+    html_colour: bool = False
 ) -> str:
     """
     Render the difference between two file contents.
@@ -93,6 +84,8 @@ def render_diff(
         line_feed_sep (str): Separator for lines in output
         include_change_type_prefix (bool): Add prefixes to indicate change types
         patch_format (bool): Use patch format for output
+        html_colour (bool): Use HTML background colors for differences
+
     Returns:
         str: Rendered difference
     """
@@ -110,7 +103,8 @@ def render_diff(
         include_removed=include_removed,
         include_added=include_added,
         include_replaced=include_replaced,
-        include_change_type_prefix=include_change_type_prefix
+        include_change_type_prefix=include_change_type_prefix,
+        html_colour=html_colour
     )
 
     def flatten(lst: List[Union[str, List[str]]]) -> str:

changedetectionio/flask_app.py

@@ -795,7 +795,7 @@ def ticker_thread_check_time_launch_checks():
             else:
                 time_schedule_limit = watch.get('time_schedule_limit')
                 logger.trace(f"{uuid} Time scheduler - Using watch settings (not global settings)")
-            tz_name = datastore.data['settings']['application'].get('scheduler_timezone_default', os.getenv('TZ', 'UTC').strip())
+            tz_name = datastore.data['settings']['application'].get('timezone', 'UTC')
 
             if time_schedule_limit and time_schedule_limit.get('enabled'):
                 try:

changedetectionio/forms.py

@@ -5,7 +5,6 @@ from wtforms.widgets.core import TimeInput
 
 from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES
 from changedetectionio.conditions.form import ConditionFormRow
-from changedetectionio.notification_service import NotificationContextData
 from changedetectionio.strtobool import strtobool
 
 from wtforms import (
@@ -470,16 +469,11 @@ class ValidateAppRiseServers(object):
         import apprise
         from .notification.apprise_plugin.assets import apprise_asset
         from .notification.apprise_plugin.custom_handlers import apprise_http_custom_handler  # noqa: F401
-        from changedetectionio.jinja2_custom import render as jinja_render
 
         apobj = apprise.Apprise(asset=apprise_asset)
 
         for server_url in field.data:
-            generic_notification_context_data = NotificationContextData()
-            # Make sure something is atleast in all those regular token fields
-            generic_notification_context_data.set_random_for_validation()
-
-            url = jinja_render(template_str=server_url.strip(), **generic_notification_context_data).strip()
+            url = server_url.strip()
             if url.startswith("#"):
                 continue
 
@@ -493,8 +487,9 @@ class ValidateJinja2Template(object):
     """
     def __call__(self, form, field):
         from changedetectionio import notification
-        from changedetectionio.jinja2_custom import create_jinja_env
+
         from jinja2 import BaseLoader, TemplateSyntaxError, UndefinedError
+        from jinja2.sandbox import ImmutableSandboxedEnvironment
         from jinja2.meta import find_undeclared_variables
         import jinja2.exceptions
 
@@ -502,11 +497,9 @@ class ValidateJinja2Template(object):
         joined_data = ' '.join(map(str, field.data)) if isinstance(field.data, list) else f"{field.data}"
 
         try:
-            # Use the shared helper to create a properly configured environment
-            jinja2_env = create_jinja_env(loader=BaseLoader)
-
-            # Add notification tokens for validation
-            jinja2_env.globals.update(NotificationContextData())
+            jinja2_env = ImmutableSandboxedEnvironment(loader=BaseLoader, extensions=['jinja2_time.TimeExtension'])
+            jinja2_env.globals.update(notification.valid_tokens)
+            # Extra validation tokens provided on the form_class(... extra_tokens={}) setup
             if hasattr(field, 'extra_notification_tokens'):
                 jinja2_env.globals.update(field.extra_notification_tokens)
 
@@ -518,7 +511,6 @@ class ValidateJinja2Template(object):
         except jinja2.exceptions.SecurityError as e:
             raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
 
-        # Check for undeclared variables
         ast = jinja2_env.parse(joined_data)
         undefined = ", ".join(find_undeclared_variables(ast))
         if undefined:
@@ -686,51 +678,6 @@ class ValidateCSSJSONXPATHInput(object):
                 except:
                     raise ValidationError("A system-error occurred when validating your jq expression")
 
-class ValidateSimpleURL:
-    """Validate that the value can be parsed by urllib.parse.urlparse() and has a scheme/netloc."""
-    def __init__(self, message=None):
-        self.message = message or "Invalid URL."
-
-    def __call__(self, form, field):
-        data = (field.data or "").strip()
-        if not data:
-            return  # empty is OK — pair with validators.Optional()
-        from urllib.parse import urlparse
-
-        parsed = urlparse(data)
-        if not parsed.scheme or not parsed.netloc:
-            raise ValidationError(self.message)
-
-class ValidateStartsWithRegex(object):
-    def __init__(self, regex, *, flags=0, message=None, allow_empty=True, split_lines=True):
-        # compile with given flags (we’ll pass re.IGNORECASE below)
-        self.pattern = re.compile(regex, flags) if isinstance(regex, str) else regex
-        self.message = message
-        self.allow_empty = allow_empty
-        self.split_lines = split_lines
-
-    def __call__(self, form, field):
-        data = field.data
-        if not data:
-            return
-
-        # normalize into list of lines
-        if isinstance(data, str) and self.split_lines:
-            lines = data.splitlines()
-        elif isinstance(data, (list, tuple)):
-            lines = data
-        else:
-            lines = [data]
-
-        for line in lines:
-            stripped = line.strip()
-            if not stripped:
-                if self.allow_empty:
-                    continue
-                raise ValidationError(self.message or "Empty value not allowed.")
-            if not self.pattern.match(stripped):
-                raise ValidationError(self.message or "Invalid value.")
-
 class quickWatchForm(Form):
     from . import processors
 
@@ -758,17 +705,9 @@ class commonSettingsForm(Form):
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
     processor = RadioField( label=u"Processor - What do you want to achieve?", choices=processors.available_processors(), default="text_json_diff")
-    scheduler_timezone_default = StringField("Default timezone for watch check scheduler", render_kw={"list": "timezones"}, validators=[validateTimeZoneName()])
+    timezone = StringField("Timezone for watch schedule", render_kw={"list": "timezones"}, validators=[validateTimeZoneName()])
     webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")])
 
-# Not true anymore but keep the validate_ hook for future use, we convert color tags
-#    def validate_notification_urls(self, field):
-#        """Validate that HTML Color format is not used with Telegram"""
-#        if self.notification_format.data == 'HTML Color' and field.data:
-#            for url in field.data:
-#                if url and ('tgram://' in url or 'discord://' in url or 'discord.com/api/webhooks' in url):
-#                    raise ValidationError('HTML Color format is not supported by Telegram and Discord. Please choose another Notification Format (Plain Text, HTML, or Markdown to HTML).')
-
 
 class importForm(Form):
     from . import processors
@@ -856,7 +795,7 @@ class processor_text_json_diff_form(commonSettingsForm):
         if not super().validate():
             return False
 
-        from changedetectionio.jinja2_custom import render as jinja_render
+        from changedetectionio.safe_jinja import render as jinja_render
         result = True
 
         # Fail form validation when a body is set for a GET
@@ -919,36 +858,23 @@ class processor_text_json_diff_form(commonSettingsForm):
     ):
         super().__init__(formdata, obj, prefix, data, meta, **kwargs)
         if kwargs and kwargs.get('default_system_settings'):
-            default_tz = kwargs.get('default_system_settings').get('application', {}).get('scheduler_timezone_default')
+            default_tz = kwargs.get('default_system_settings').get('application', {}).get('timezone')
             if default_tz:
                 self.time_schedule_limit.form.timezone.render_kw['placeholder'] = default_tz
 
 
 
 class SingleExtraProxy(Form):
+
     # maybe better to set some <script>var..
     proxy_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
-    proxy_url = StringField('Proxy URL', [
-        validators.Optional(),
-        ValidateStartsWithRegex(
-            regex=r'^(https?|socks5)://',  # ✅ main pattern
-            flags=re.IGNORECASE,  # ✅ makes it case-insensitive
-            message='Proxy URLs must start with http://, https:// or socks5://',
-        ),
-        ValidateSimpleURL()
-    ], render_kw={"placeholder": "socks5:// or regular proxy http://user:pass@...:3128", "size":50})
+    proxy_url = StringField('Proxy URL', [validators.Optional()], render_kw={"placeholder": "socks5:// or regular proxy http://user:pass@...:3128", "size":50})
+    # @todo do the validation here instead
 
 class SingleExtraBrowser(Form):
     browser_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
-    browser_connection_url = StringField('Browser connection URL', [
-        validators.Optional(),
-        ValidateStartsWithRegex(
-            regex=r'^(wss?|ws)://',
-            flags=re.IGNORECASE,
-            message='Browser URLs must start with wss:// or ws://'
-        ),
-        ValidateSimpleURL()
-    ], render_kw={"placeholder": "wss://brightdata... wss://oxylabs etc", "size":50})
+    browser_connection_url = StringField('Browser connection URL', [validators.Optional()], render_kw={"placeholder": "wss://brightdata... wss://oxylabs etc", "size":50})
+    # @todo do the validation here instead
 
 class DefaultUAInputForm(Form):
     html_requests = StringField('Plaintext requests', validators=[validators.Optional()], render_kw={"placeholder": "<default>"})
@@ -959,7 +885,7 @@ class DefaultUAInputForm(Form):
 class globalSettingsRequestForm(Form):
     time_between_check = RequiredFormField(TimeBetweenCheckForm)
     time_schedule_limit = FormField(ScheduleLimitForm)
-    proxy = RadioField('Default proxy')
+    proxy = RadioField('Proxy')
     jitter_seconds = IntegerField('Random jitter seconds ± check',
                                   render_kw={"style": "width: 5em;"},
                                   validators=[validators.NumberRange(min=0, message="Should contain zero or more seconds")])
@@ -968,12 +894,7 @@ class globalSettingsRequestForm(Form):
                           render_kw={"style": "width: 5em;"},
                           validators=[validators.NumberRange(min=1, max=50,
                                                              message="Should be between 1 and 50")])
-
-    timeout = IntegerField('Requests timeout in seconds',
-                           render_kw={"style": "width: 5em;"},
-                           validators=[validators.NumberRange(min=1, max=999,
-                                                              message="Should be between 1 and 999")])
-
+    
     extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5)
     extra_browsers = FieldList(FormField(SingleExtraBrowser), min_entries=5)
 
@@ -1019,10 +940,6 @@ class globalSettingsApplicationForm(commonSettingsForm):
     strip_ignored_lines = BooleanField('Strip ignored lines')
     rss_hide_muted_watches = BooleanField('Hide muted watches from RSS feed', default=True,
                                       validators=[validators.Optional()])
-
-    rss_reader_mode = BooleanField('RSS reader mode ', default=False,
-                                      validators=[validators.Optional()])
-
     filter_failure_notification_threshold_attempts = IntegerField('Number of times the filter can be missing before sending a notification',
                                                                   render_kw={"style": "width: 5em;"},
                                                                   validators=[validators.NumberRange(min=0,

changedetectionio/html_tools.py

@@ -185,21 +185,8 @@ def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False
     tree = html.fromstring(bytes(html_content, encoding='utf-8'), parser=parser)
     html_block = ""
 
-    # Build namespace map for XPath queries
-    namespaces = {'re': 'http://exslt.org/regular-expressions'}
-
-    # Handle default namespace in documents (common in RSS/Atom feeds, but can occur in any XML)
-    # XPath spec: unprefixed element names have no namespace, not the default namespace
-    # Solution: Register the default namespace with empty string prefix in elementpath
-    # This is primarily for RSS/Atom feeds but works for any XML with default namespace
-    if hasattr(tree, 'nsmap') and tree.nsmap and None in tree.nsmap:
-        # Register the default namespace with empty string prefix for elementpath
-        # This allows //title to match elements in the default namespace
-        namespaces[''] = tree.nsmap[None]
-
-    r = elementpath.select(tree, xpath_filter.strip(), namespaces=namespaces, parser=XPath3Parser)
-    #@note: //title/text() now works with default namespaces (fixed by registering '' prefix)
-    #@note: //title/text() wont work where <title>CDATA.. (use cdata_in_document_to_text first)
+    r = elementpath.select(tree, xpath_filter.strip(), namespaces={'re': 'http://exslt.org/regular-expressions'}, parser=XPath3Parser)
+    #@note: //title/text() wont work where <title>CDATA..
 
     if type(r) != list:
         r = [r]
@@ -234,19 +221,8 @@ def xpath1_filter(xpath_filter, html_content, append_pretty_line_formatting=Fals
     tree = html.fromstring(bytes(html_content, encoding='utf-8'), parser=parser)
     html_block = ""
 
-    # Build namespace map for XPath queries
-    namespaces = {'re': 'http://exslt.org/regular-expressions'}
-
-    # NOTE: lxml's native xpath() does NOT support empty string prefix for default namespace
-    # For documents with default namespace (RSS/Atom feeds), users must use:
-    #   - local-name(): //*[local-name()='title']/text()
-    #   - Or use xpath_filter (not xpath1_filter) which supports default namespaces
-    # XPath spec: unprefixed element names have no namespace, not the default namespace
-
-    r = tree.xpath(xpath_filter.strip(), namespaces=namespaces)
-    #@note: xpath1 (lxml) does NOT automatically handle default namespaces
-    #@note: Use //*[local-name()='element'] or switch to xpath_filter for default namespace support
-    #@note: //title/text() wont work where <title>CDATA.. (use cdata_in_document_to_text first)
+    r = tree.xpath(xpath_filter.strip(), namespaces={'re': 'http://exslt.org/regular-expressions'})
+    #@note: //title/text() wont work where <title>CDATA..
 
     for element in r:
         # When there's more than 1 match, then add the suffix to separate each line
@@ -432,9 +408,6 @@ def strip_ignore_text(content, wordlist, mode="content"):
     ignored_lines = []
 
     for k in wordlist:
-        # Skip empty strings to avoid matching everything
-        if not k or not k.strip():
-            continue
         # Is it a regex?
         res = re.search(PERL_STYLE_REGEX, k, re.IGNORECASE)
         if res:

changedetectionio/jinja2_custom/__init__.py

@@ -1,22 +0,0 @@
-"""
-Jinja2 custom extensions and safe rendering utilities.
-"""
-from .extensions.TimeExtension import TimeExtension
-from .safe_jinja import (
-    render,
-    render_fully_escaped,
-    create_jinja_env,
-    JINJA2_MAX_RETURN_PAYLOAD_SIZE,
-    DEFAULT_JINJA2_EXTENSIONS,
-)
-from .plugins.regex import regex_replace
-
-__all__ = [
-    'TimeExtension',
-    'render',
-    'render_fully_escaped',
-    'create_jinja_env',
-    'JINJA2_MAX_RETURN_PAYLOAD_SIZE',
-    'DEFAULT_JINJA2_EXTENSIONS',
-    'regex_replace',
-]

changedetectionio/jinja2_custom/extensions/TimeExtension.py

@@ -1,221 +0,0 @@
-"""
-Jinja2 TimeExtension - Custom date/time handling for templates.
-
-This extension provides the {% now %} tag for Jinja2 templates, offering timezone-aware
-date/time formatting with support for time offsets.
-
-Why This Extension Exists:
-    The Arrow library has a now() function (arrow.now()), but Jinja2 templates cannot
-    directly call Python functions - they need extensions or filters to expose functionality.
-
-    This TimeExtension serves as a Jinja2-to-Arrow bridge that:
-
-    1. Makes Arrow accessible in templates - Jinja2 requires registering functions/tags
-       through extensions. You cannot use arrow.now() directly in a template.
-
-    2. Provides template-friendly syntax - Instead of complex Python code, you get clean tags:
-       {% now 'UTC' %}
-       {% now 'UTC' + 'hours=2' %}
-       {% now 'Europe/London', '%Y-%m-%d' %}
-
-    3. Adds convenience features on top of Arrow:
-       - Default timezone from environment variable (TZ) or config
-       - Default datetime format configuration
-       - Offset syntax parsing: 'hours=2,minutes=30' → shift(hours=2, minutes=30)
-       - Empty string timezone support to use configured defaults
-
-    4. Maintains security - Works within Jinja2's sandboxed environment so users
-       cannot access arbitrary Python code or objects.
-
-    Essentially, this is a Jinja2 wrapper around arrow.now() and arrow.shift() that
-    provides user-friendly template syntax while maintaining security.
-
-Basic Usage:
-    {% now 'UTC' %}
-    # Output: Wed, 09 Dec 2015 23:33:01
-
-Custom Format:
-    {% now 'UTC', '%Y-%m-%d %H:%M:%S' %}
-    # Output: 2015-12-09 23:33:01
-
-Timezone Support:
-    {% now 'America/New_York' %}
-    {% now 'Europe/London' %}
-    {% now '' %}  # Uses default timezone from environment.default_timezone
-
-Time Offsets (Addition):
-    {% now 'UTC' + 'hours=2' %}
-    {% now 'UTC' + 'hours=2,minutes=30' %}
-    {% now 'UTC' + 'days=1,hours=2,minutes=15,seconds=10' %}
-
-Time Offsets (Subtraction):
-    {% now 'UTC' - 'minutes=11' %}
-    {% now 'UTC' - 'days=2,minutes=33,seconds=1' %}
-
-Time Offsets with Custom Format:
-    {% now 'UTC' + 'hours=2', '%Y-%m-%d %H:%M:%S' %}
-    # Output: 2015-12-10 01:33:01
-
-Weekday Support (for finding next/previous weekday):
-    {% now 'UTC' + 'weekday=0' %}  # Next Monday (0=Monday, 6=Sunday)
-    {% now 'UTC' + 'weekday=4' %}  # Next Friday
-
-Configuration:
-    - Default timezone: Set via TZ environment variable or override environment.default_timezone
-    - Default format: '%a, %d %b %Y %H:%M:%S' (can be overridden via environment.datetime_format)
-
-Environment Customization:
-    from changedetectionio.jinja2_custom import create_jinja_env
-
-    jinja2_env = create_jinja_env()
-    jinja2_env.default_timezone = 'America/New_York'  # Override default timezone
-    jinja2_env.datetime_format = '%Y-%m-%d %H:%M'      # Override default format
-
-Supported Offset Parameters:
-    - years, months, weeks, days
-    - hours, minutes, seconds, microseconds
-    - weekday (0=Monday through 6=Sunday, must be integer)
-
-Note:
-    This extension uses the Arrow library for timezone-aware datetime handling.
-    All timezone names should be valid IANA timezone identifiers (e.g., 'America/New_York').
-"""
-import arrow
-
-from jinja2 import nodes
-from jinja2.ext import Extension
-import os
-
-class TimeExtension(Extension):
-    """
-    Jinja2 Extension providing the {% now %} tag for timezone-aware date/time rendering.
-
-    This extension adds two attributes to the Jinja2 environment:
-    - datetime_format: Default strftime format string (default: '%a, %d %b %Y %H:%M:%S')
-    - default_timezone: Default timezone for rendering (default: TZ env var or 'UTC')
-
-    Both can be overridden after environment creation by setting the attributes directly.
-    """
-
-    tags = {'now'}
-
-    def __init__(self, environment):
-        """Jinja2 Extension constructor."""
-        super().__init__(environment)
-
-        environment.extend(
-            datetime_format='%a, %d %b %Y %H:%M:%S',
-            default_timezone=os.getenv('TZ', 'UTC').strip()
-        )
-
-    def _datetime(self, timezone, operator, offset, datetime_format):
-        """
-        Get current datetime with time offset applied.
-
-        Args:
-            timezone: IANA timezone identifier (e.g., 'UTC', 'America/New_York') or empty string for default
-            operator: '+' for addition or '-' for subtraction
-            offset: Comma-separated offset parameters (e.g., 'hours=2,minutes=30')
-            datetime_format: strftime format string or None to use environment default
-
-        Returns:
-            Formatted datetime string with offset applied
-
-        Example:
-            _datetime('UTC', '+', 'hours=2,minutes=30', '%Y-%m-%d %H:%M:%S')
-            # Returns current time + 2.5 hours
-        """
-        # Use default timezone if none specified
-        if not timezone or timezone == '':
-            timezone = self.environment.default_timezone
-
-        d = arrow.now(timezone)
-
-        # parse shift params from offset and include operator
-        shift_params = {}
-        for param in offset.split(','):
-            interval, value = param.split('=')
-            shift_params[interval.strip()] = float(operator + value.strip())
-
-        # Fix weekday parameter can not be float
-        if 'weekday' in shift_params:
-            shift_params['weekday'] = int(shift_params['weekday'])
-
-        d = d.shift(**shift_params)
-
-        if datetime_format is None:
-            datetime_format = self.environment.datetime_format
-        return d.strftime(datetime_format)
-
-    def _now(self, timezone, datetime_format):
-        """
-        Get current datetime without any offset.
-
-        Args:
-            timezone: IANA timezone identifier (e.g., 'UTC', 'America/New_York') or empty string for default
-            datetime_format: strftime format string or None to use environment default
-
-        Returns:
-            Formatted datetime string for current time
-
-        Example:
-            _now('America/New_York', '%Y-%m-%d %H:%M:%S')
-            # Returns current time in New York timezone
-        """
-        # Use default timezone if none specified
-        if not timezone or timezone == '':
-            timezone = self.environment.default_timezone
-
-        if datetime_format is None:
-            datetime_format = self.environment.datetime_format
-        return arrow.now(timezone).strftime(datetime_format)
-
-    def parse(self, parser):
-        """
-        Parse the {% now %} tag and generate appropriate AST nodes.
-
-        This method is called by Jinja2 when it encounters a {% now %} tag.
-        It parses the tag syntax and determines whether to call _now() or _datetime()
-        based on whether offset operations (+ or -) are present.
-
-        Supported syntax:
-            {% now 'timezone' %}                              -> calls _now()
-            {% now 'timezone', 'format' %}                    -> calls _now()
-            {% now 'timezone' + 'offset' %}                   -> calls _datetime()
-            {% now 'timezone' + 'offset', 'format' %}         -> calls _datetime()
-            {% now 'timezone' - 'offset', 'format' %}         -> calls _datetime()
-
-        Args:
-            parser: Jinja2 parser instance
-
-        Returns:
-            nodes.Output: AST output node containing the formatted datetime string
-        """
-        lineno = next(parser.stream).lineno
-
-        node = parser.parse_expression()
-
-        if parser.stream.skip_if('comma'):
-            datetime_format = parser.parse_expression()
-        else:
-            datetime_format = nodes.Const(None)
-
-        if isinstance(node, nodes.Add):
-            call_method = self.call_method(
-                '_datetime',
-                [node.left, nodes.Const('+'), node.right, datetime_format],
-                lineno=lineno,
-            )
-        elif isinstance(node, nodes.Sub):
-            call_method = self.call_method(
-                '_datetime',
-                [node.left, nodes.Const('-'), node.right, datetime_format],
-                lineno=lineno,
-            )
-        else:
-            call_method = self.call_method(
-                '_now',
-                [node, datetime_format],
-                lineno=lineno,
-            )
-        return nodes.Output([call_method], lineno=lineno)

changedetectionio/jinja2_custom/plugins/__init__.py

@@ -1,6 +0,0 @@
-"""
-Jinja2 custom filter plugins for changedetection.io
-"""
-from .regex import regex_replace
-
-__all__ = ['regex_replace']

changedetectionio/jinja2_custom/plugins/regex.py

@@ -1,98 +0,0 @@
-"""
-Regex filter plugin for Jinja2 templates.
-
-Provides regex_replace filter for pattern-based string replacements in templates.
-"""
-import re
-import signal
-from loguru import logger
-
-
-def regex_replace(value: str, pattern: str, replacement: str = '', count: int = 0) -> str:
-    """
-    Replace occurrences of a regex pattern in a string.
-
-    Security: Protected against ReDoS (Regular Expression Denial of Service) attacks:
-    - Limits input value size to prevent excessive processing
-    - Uses timeout mechanism to prevent runaway regex operations
-    - Validates pattern complexity to prevent catastrophic backtracking
-
-    Args:
-        value: The input string to perform replacements on
-        pattern: The regex pattern to search for
-        replacement: The replacement string (default: '')
-        count: Maximum number of replacements (0 = replace all, default: 0)
-
-    Returns:
-        String with replacements applied, or original value on error
-
-    Example:
-        {{ "hello world" | regex_replace("world", "universe") }}
-        {{ diff | regex_replace("<td>([^<]+)</td><td>([^<]+)</td>", "Label1: \\1\\nLabel2: \\2") }}
-
-    Security limits:
-        - Maximum input size: 10MB
-        - Maximum pattern length: 500 characters
-        - Operation timeout: 10 seconds
-        - Dangerous nested quantifier patterns are rejected
-    """
-    # Security limits
-    MAX_INPUT_SIZE = 1024 * 1024 * 10 # 10MB max input size
-    MAX_PATTERN_LENGTH = 500  # Maximum regex pattern length
-    REGEX_TIMEOUT_SECONDS = 10  # Maximum time for regex operation
-
-    # Validate input sizes
-    value_str = str(value)
-    if len(value_str) > MAX_INPUT_SIZE:
-        logger.warning(f"regex_replace: Input too large ({len(value_str)} bytes), truncating")
-        value_str = value_str[:MAX_INPUT_SIZE]
-
-    if len(pattern) > MAX_PATTERN_LENGTH:
-        logger.warning(f"regex_replace: Pattern too long ({len(pattern)} chars), rejecting")
-        return value_str
-
-    # Check for potentially dangerous patterns (basic checks)
-    # Nested quantifiers like (a+)+ can cause catastrophic backtracking
-    dangerous_patterns = [
-        r'\([^)]*\+[^)]*\)\+',  # (x+)+
-        r'\([^)]*\*[^)]*\)\+',  # (x*)+
-        r'\([^)]*\+[^)]*\)\*',  # (x+)*
-        r'\([^)]*\*[^)]*\)\*',  # (x*)*
-    ]
-
-    for dangerous in dangerous_patterns:
-        if re.search(dangerous, pattern):
-            logger.warning(f"regex_replace: Potentially dangerous pattern detected: {pattern}")
-            return value_str
-
-    def timeout_handler(signum, frame):
-        raise TimeoutError("Regex operation timed out")
-
-    try:
-        # Set up timeout for regex operation (Unix-like systems only)
-        # This prevents ReDoS attacks
-        old_handler = None
-        if hasattr(signal, 'SIGALRM'):
-            old_handler = signal.signal(signal.SIGALRM, timeout_handler)
-            signal.alarm(REGEX_TIMEOUT_SECONDS)
-
-        try:
-            result = re.sub(pattern, replacement, value_str, count=count)
-        finally:
-            # Cancel the alarm
-            if hasattr(signal, 'SIGALRM'):
-                signal.alarm(0)
-                if old_handler is not None:
-                    signal.signal(signal.SIGALRM, old_handler)
-
-        return result
-
-    except TimeoutError:
-        logger.error(f"regex_replace: Regex operation timed out - possible ReDoS attack. Pattern: {pattern}")
-        return value_str
-    except re.error as e:
-        logger.warning(f"regex_replace: Invalid regex pattern: {e}")
-        return value_str
-    except Exception as e:
-        logger.error(f"regex_replace: Unexpected error: {e}")
-        return value_str

changedetectionio/jinja2_custom/safe_jinja.py

@@ -1,58 +0,0 @@
-"""
-Safe Jinja2 render with max payload sizes
-
-See https://jinja.palletsprojects.com/en/3.1.x/sandbox/#security-considerations
-"""
-
-import jinja2.sandbox
-import typing as t
-import os
-from .extensions.TimeExtension import TimeExtension
-from .plugins import regex_replace
-
-JINJA2_MAX_RETURN_PAYLOAD_SIZE = 1024 * int(os.getenv("JINJA2_MAX_RETURN_PAYLOAD_SIZE_KB", 1024 * 10))
-
-# Default extensions - can be overridden in create_jinja_env()
-DEFAULT_JINJA2_EXTENSIONS = [TimeExtension]
-
-def create_jinja_env(extensions=None, **kwargs) -> jinja2.sandbox.ImmutableSandboxedEnvironment:
-    """
-    Create a sandboxed Jinja2 environment with our custom extensions and default timezone.
-
-    Args:
-        extensions: List of extension classes to use (defaults to DEFAULT_JINJA2_EXTENSIONS)
-        **kwargs: Additional arguments to pass to ImmutableSandboxedEnvironment
-
-    Returns:
-        Configured Jinja2 environment
-    """
-    if extensions is None:
-        extensions = DEFAULT_JINJA2_EXTENSIONS
-
-    jinja2_env = jinja2.sandbox.ImmutableSandboxedEnvironment(
-        extensions=extensions,
-        **kwargs
-    )
-
-    # Get default timezone from environment variable
-    default_timezone = os.getenv('TZ', 'UTC').strip()
-    jinja2_env.default_timezone = default_timezone
-
-    # Register custom filters
-    jinja2_env.filters['regex_replace'] = regex_replace
-
-    return jinja2_env
-
-
-# This is used for notifications etc, so actually it's OK to send custom HTML such as <a href> etc, but it should limit what data is available.
-# (Which also limits available functions that could be called)
-def render(template_str, **args: t.Any) -> str:
-    jinja2_env = create_jinja_env()
-    output = jinja2_env.from_string(template_str).render(args)
-    return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]
-
-def render_fully_escaped(content):
-    env = jinja2.sandbox.ImmutableSandboxedEnvironment(autoescape=True)
-    template = env.from_string("{{ some_html|e }}")
-    return template.render(some_html=content)
-

changedetectionio/model/App.py

@@ -55,12 +55,11 @@ class model(dict):
                     'rss_access_token': None,
                     'rss_content_format': RSS_FORMAT_TYPES[0][0],
                     'rss_hide_muted_watches': True,
-                    'rss_reader_mode': False,
-                    'scheduler_timezone_default': None,  # Default IANA timezone name
                     'schema_version' : 0,
                     'shared_diff_access': False,
                     'strip_ignored_lines': False,
                     'tags': {}, #@todo use Tag.model initialisers
+                    'timezone': None, # Default IANA timezone name
                     'webdriver_delay': None , # Extra delay in seconds before extracting text
                     'ui': {
                         'use_page_title_in_list': True,

changedetectionio/model/Watch.py

@@ -1,15 +1,14 @@
 from blinker import signal
 
 from changedetectionio.strtobool import strtobool
-from changedetectionio.jinja2_custom import render as jinja_render
+from changedetectionio.safe_jinja import render as jinja_render
 from . import watch_base
 import os
 import re
 from pathlib import Path
 from loguru import logger
 
-from .. import jinja2_custom as safe_jinja
-from ..diff import ADDED_PLACEMARKER_OPEN
+from .. import safe_jinja
 from ..html_tools import TRANSLATE_WHITESPACE_TABLE
 
 # Allowable protocols, protects against javascript: etc
@@ -90,8 +89,9 @@ class model(watch_base):
                 ready_url = jinja_render(template_str=url)
             except Exception as e:
                 logger.critical(f"Invalid URL template for: '{url}' - {str(e)}")
-                from flask import flash, url_for
-                from markupsafe import Markup
+                from flask import (
+                    flash, Markup, url_for
+                )
                 message = Markup('<a href="{}#general">The URL {} is invalid and cannot be used, click to edit</a>'.format(
                     url_for('ui.ui_edit.edit_page', uuid=self.get('uuid')), self.get('url', '')))
                 flash(message, 'error')

changedetectionio/notification/__init__.py

@@ -1,5 +1,6 @@
 from changedetectionio.model import default_notification_format_for_watch
 
+ult_notification_format_for_watch = 'System default'
 default_notification_format = 'HTML Color'
 default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
 default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
@@ -7,11 +8,28 @@ default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
 # The values (markdown etc) are from apprise NotifyFormat,
 # But to avoid importing the whole heavy module just use the same strings here.
 valid_notification_formats = {
-    'Plain Text': 'text',
+    'Text': 'text',
+    'Markdown': 'markdown',
     'HTML': 'html',
     'HTML Color': 'htmlcolor',
-    'Markdown to HTML': 'markdown',
     # Used only for editing a watch (not for global)
     default_notification_format_for_watch: default_notification_format_for_watch
 }
 
+
+valid_tokens = {
+    'base_url': '',
+    'current_snapshot': '',
+    'diff': '',
+    'diff_added': '',
+    'diff_full': '',
+    'diff_patch': '',
+    'diff_removed': '',
+    'diff_url': '',
+    'preview_url': '',
+    'triggered_text': '',
+    'watch_tag': '',
+    'watch_title': '',
+    'watch_url': '',
+    'watch_uuid': '',
+}

changedetectionio/notification/apprise_plugin/custom_handlers.py

@@ -70,7 +70,6 @@ def apprise_http_custom_handler(
     title: str,
     notify_type: str,
     meta: dict,
-    body_format: str = None,
     *args,
     **kwargs,
 ) -> bool:

changedetectionio/notification/apprise_plugin/discord.py

@@ -1,286 +0,0 @@
-"""
-Custom Discord plugin for changedetection.io
-Extends Apprise's Discord plugin to support custom colored embeds for removed/added content
-"""
-from apprise.plugins.discord import NotifyDiscord
-from apprise.decorators import notify
-from apprise.common import NotifyFormat
-from loguru import logger
-
-# Import placeholders from changedetection's diff module
-from ...diff import (
-    REMOVED_PLACEMARKER_OPEN,
-    REMOVED_PLACEMARKER_CLOSED,
-    ADDED_PLACEMARKER_OPEN,
-    ADDED_PLACEMARKER_CLOSED,
-    CHANGED_PLACEMARKER_OPEN,
-    CHANGED_PLACEMARKER_CLOSED,
-    CHANGED_INTO_PLACEMARKER_OPEN,
-    CHANGED_INTO_PLACEMARKER_CLOSED,
-)
-
-# Discord embed sidebar colors for different change types
-DISCORD_COLOR_UNCHANGED = 8421504   # Gray (#808080)
-DISCORD_COLOR_REMOVED = 16711680    # Red (#FF0000)
-DISCORD_COLOR_ADDED = 65280         # Green (#00FF00)
-DISCORD_COLOR_CHANGED = 16753920    # Orange (#FFA500)
-DISCORD_COLOR_CHANGED_INTO = 3447003  # Blue (#5865F2 - Discord blue)
-DISCORD_COLOR_WARNING = 16776960    # Yellow (#FFFF00)
-
-
-class NotifyDiscordCustom(NotifyDiscord):
-    """
-    Custom Discord notification handler that supports multiple colored embeds
-    for showing removed (red) and added (green) content separately.
-    """
-
-    def send(self, body, title="", notify_type=None, attach=None, **kwargs):
-        """
-        Override send method to create custom embeds with red/green colors
-        for removed/added content when placeholders are present.
-        """
-
-        # Check if body contains our diff placeholders
-        has_removed = REMOVED_PLACEMARKER_OPEN in body
-        has_added = ADDED_PLACEMARKER_OPEN in body
-        has_changed = CHANGED_PLACEMARKER_OPEN in body
-        has_changed_into = CHANGED_INTO_PLACEMARKER_OPEN in body
-
-        # If we have diff placeholders and we're in markdown/html format, create custom embeds
-        if (has_removed or has_added or has_changed or has_changed_into) and self.notify_format in (NotifyFormat.MARKDOWN, NotifyFormat.HTML):
-            return self._send_with_colored_embeds(body, title, notify_type, attach, **kwargs)
-
-        # Otherwise, use the parent class's default behavior
-        return super().send(body, title, notify_type, attach, **kwargs)
-
-    def _send_with_colored_embeds(self, body, title, notify_type, attach, **kwargs):
-        """
-        Send Discord message with embeds in the original diff order.
-        Preserves the sequence: unchanged -> removed -> added -> unchanged, etc.
-        """
-        from datetime import datetime, timezone
-
-        payload = {
-            "tts": self.tts,
-            "wait": self.tts is False,
-        }
-
-        if self.flags:
-            payload["flags"] = self.flags
-
-        # Acquire image_url
-        image_url = self.image_url(notify_type)
-
-        if self.avatar and (image_url or self.avatar_url):
-            payload["avatar_url"] = self.avatar_url if self.avatar_url else image_url
-
-        if self.user:
-            payload["username"] = self.user
-
-        # Associate our thread_id with our message
-        params = {"thread_id": self.thread_id} if self.thread_id else None
-
-        # Build embeds array preserving order
-        embeds = []
-
-        # Add title as plain bold text in message content (not an embed)
-        if title:
-            payload["content"] = f"**{title}**"
-
-        # Parse the body into ordered chunks
-        chunks = self._parse_body_into_chunks(body)
-
-        # Discord limits:
-        # - Max 10 embeds per message
-        # - Max 6000 characters total across all embeds
-        # - Max 4096 characters per embed description
-        max_embeds = 10
-        max_total_chars = 6000
-        max_embed_description = 4096
-
-        # All 10 embed slots are available for content
-        max_content_embeds = max_embeds
-
-        # Start character count
-        total_chars = 0
-
-        # Create embeds from chunks in order (no titles, just color coding)
-        for chunk_type, content in chunks:
-            if not content.strip():
-                continue
-
-            # Truncate individual embed description if needed
-            if len(content) > max_embed_description:
-                content = content[:max_embed_description - 3] + "..."
-
-            # Check if we're approaching the embed count limit
-            # We need room for the warning embed, so stop at max_content_embeds - 1
-            current_content_embeds = len(embeds)
-            if current_content_embeds >= max_content_embeds - 1:
-                # Add a truncation notice (this will be the 10th embed)
-                embeds.append({
-                    "description": "⚠️ Content truncated (Discord 10 embed limit reached) - Tip: Select 'Plain Text' or 'HTML' format for longer diffs",
-                    "color": DISCORD_COLOR_WARNING,
-                })
-                break
-
-            # Check if adding this embed would exceed total character limit
-            if total_chars + len(content) > max_total_chars:
-                # Add a truncation notice
-                remaining_chars = max_total_chars - total_chars
-                if remaining_chars > 100:
-                    # Add partial content if we have room
-                    truncated_content = content[:remaining_chars - 100] + "..."
-                    embeds.append({
-                        "description": truncated_content,
-                        "color": (DISCORD_COLOR_UNCHANGED if chunk_type == "unchanged"
-                                 else DISCORD_COLOR_REMOVED if chunk_type == "removed"
-                                 else DISCORD_COLOR_ADDED),
-                    })
-                embeds.append({
-                    "description": "⚠️ Content truncated (Discord 6000 char limit reached)\nTip: Select 'Plain Text' or 'HTML' format for longer diffs",
-                    "color": DISCORD_COLOR_WARNING,
-                })
-                break
-
-            if chunk_type == "unchanged":
-                embeds.append({
-                    "description": content,
-                    "color": DISCORD_COLOR_UNCHANGED,
-                })
-            elif chunk_type == "removed":
-                embeds.append({
-                    "description": content,
-                    "color": DISCORD_COLOR_REMOVED,
-                })
-            elif chunk_type == "added":
-                embeds.append({
-                    "description": content,
-                    "color": DISCORD_COLOR_ADDED,
-                })
-            elif chunk_type == "changed":
-                # Changed (old value) - use orange to distinguish from pure removal
-                embeds.append({
-                    "description": content,
-                    "color": DISCORD_COLOR_CHANGED,
-                })
-            elif chunk_type == "changed_into":
-                # Changed into (new value) - use blue to distinguish from pure addition
-                embeds.append({
-                    "description": content,
-                    "color": DISCORD_COLOR_CHANGED_INTO,
-                })
-
-            total_chars += len(content)
-
-        if embeds:
-            payload["embeds"] = embeds
-
-        # Send the payload using parent's _send method
-        if not self._send(payload, params=params):
-            return False
-
-        # Handle attachments if present
-        if attach and self.attachment_support:
-            payload.update({
-                "tts": False,
-                "wait": True,
-            })
-            payload.pop("embeds", None)
-            payload.pop("content", None)
-            payload.pop("allow_mentions", None)
-
-            for attachment in attach:
-                self.logger.info(f"Posting Discord Attachment {attachment.name}")
-                if not self._send(payload, params=params, attach=attachment):
-                    return False
-
-        return True
-
-    def _parse_body_into_chunks(self, body):
-        """
-        Parse the body into ordered chunks of (type, content) tuples.
-        Types: "unchanged", "removed", "added", "changed", "changed_into"
-        Preserves the original order of the diff.
-        """
-        chunks = []
-        position = 0
-
-        while position < len(body):
-            # Find the next marker
-            next_removed = body.find(REMOVED_PLACEMARKER_OPEN, position)
-            next_added = body.find(ADDED_PLACEMARKER_OPEN, position)
-            next_changed = body.find(CHANGED_PLACEMARKER_OPEN, position)
-            next_changed_into = body.find(CHANGED_INTO_PLACEMARKER_OPEN, position)
-
-            # Determine which marker comes first
-            if next_removed == -1 and next_added == -1 and next_changed == -1 and next_changed_into == -1:
-                # No more markers, rest is unchanged
-                if position < len(body):
-                    chunks.append(("unchanged", body[position:]))
-                break
-
-            # Find the earliest marker
-            next_marker_pos = None
-            next_marker_type = None
-
-            # Compare all marker positions to find the earliest
-            markers = []
-            if next_removed != -1:
-                markers.append((next_removed, "removed"))
-            if next_added != -1:
-                markers.append((next_added, "added"))
-            if next_changed != -1:
-                markers.append((next_changed, "changed"))
-            if next_changed_into != -1:
-                markers.append((next_changed_into, "changed_into"))
-
-            if markers:
-                next_marker_pos, next_marker_type = min(markers, key=lambda x: x[0])
-
-            # Add unchanged content before the marker
-            if next_marker_pos > position:
-                chunks.append(("unchanged", body[position:next_marker_pos]))
-
-            # Find the closing marker
-            if next_marker_type == "removed":
-                open_marker = REMOVED_PLACEMARKER_OPEN
-                close_marker = REMOVED_PLACEMARKER_CLOSED
-            elif next_marker_type == "added":
-                open_marker = ADDED_PLACEMARKER_OPEN
-                close_marker = ADDED_PLACEMARKER_CLOSED
-            elif next_marker_type == "changed":
-                open_marker = CHANGED_PLACEMARKER_OPEN
-                close_marker = CHANGED_PLACEMARKER_CLOSED
-            else:  # changed_into
-                open_marker = CHANGED_INTO_PLACEMARKER_OPEN
-                close_marker = CHANGED_INTO_PLACEMARKER_CLOSED
-
-            close_pos = body.find(close_marker, next_marker_pos)
-
-            if close_pos == -1:
-                # No closing marker, take rest as this type
-                content = body[next_marker_pos + len(open_marker):]
-                chunks.append((next_marker_type, content))
-                break
-            else:
-                # Extract content between markers
-                content = body[next_marker_pos + len(open_marker):close_pos]
-                chunks.append((next_marker_type, content))
-                position = close_pos + len(close_marker)
-
-        return chunks
-
-
-# Register the custom Discord handler with Apprise
-# This will override the built-in discord:// handler
-@notify(on="discord")
-def discord_custom_wrapper(body, title, notify_type, meta, body_format=None, *args, **kwargs):
-    """
-    Wrapper function to make the custom Discord handler work with Apprise's decorator system.
-    Note: This decorator approach may not work for overriding built-in plugins.
-    The class-based approach above is the proper way to extend NotifyDiscord.
-    """
-    logger.info("Custom Discord handler called")
-    # This is here for potential future use with decorator-based registration
-    return True

changedetectionio/notification/handler.py

@@ -1,243 +1,33 @@
 
 import time
 import apprise
-from apprise import NotifyFormat
 from loguru import logger
-from urllib.parse import urlparse
 from .apprise_plugin.assets import apprise_asset, APPRISE_AVATAR_URL
-from .apprise_plugin.custom_handlers import SUPPORTED_HTTP_METHODS
-from ..diff import HTML_REMOVED_STYLE, REMOVED_PLACEMARKER_OPEN, REMOVED_PLACEMARKER_CLOSED, ADDED_PLACEMARKER_OPEN, HTML_ADDED_STYLE, \
-    ADDED_PLACEMARKER_CLOSED, CHANGED_INTO_PLACEMARKER_OPEN, CHANGED_INTO_PLACEMARKER_CLOSED, CHANGED_PLACEMARKER_OPEN, \
-    CHANGED_PLACEMARKER_CLOSED
-from ..notification_service import NotificationContextData
 
-
-def markup_text_links_to_html(body):
-    """
-    Convert plaintext to HTML with clickable links.
-    Uses Jinja2's escape and Markup for XSS safety.
-    """
-    from linkify_it import LinkifyIt
-    from markupsafe import Markup, escape
-
-    linkify = LinkifyIt()
-
-    # Match URLs in the ORIGINAL text (before escaping)
-    matches = linkify.match(body)
-
-    if not matches:
-        # No URLs, just escape everything
-        return Markup(escape(body))
-
-    result = []
-    last_index = 0
-
-    # Process each URL match
-    for match in matches:
-        # Add escaped text before the URL
-        if match.index > last_index:
-            text_part = body[last_index:match.index]
-            result.append(escape(text_part))
-
-        # Add the link with escaped URL (both in href and display)
-        url = match.url
-        result.append(Markup(f'<a href="{escape(url)}">{escape(url)}</a>'))
-
-        last_index = match.last_index
-
-    # Add remaining escaped text
-    if last_index < len(body):
-        result.append(escape(body[last_index:]))
-
-    # Join all parts
-    return str(Markup(''.join(str(part) for part in result)))
-
-def notification_format_align_with_apprise(n_format : str):
-    """
-    Correctly align changedetection's formats with apprise's formats
-    Probably these are the same - but good to be sure.
-    These set the expected OUTPUT format type
-    :param n_format:
-    :return:
-    """
-
-    if n_format.lower().startswith('html'):
-        # Apprise only knows 'html' not 'htmlcolor' etc, which shouldnt matter here
-        n_format = NotifyFormat.HTML.value
-    elif n_format.lower().startswith('markdown'):
-        # probably the same but just to be safe
-        n_format = NotifyFormat.MARKDOWN.value
-    elif n_format.lower().startswith('text'):
-        # probably the same but just to be safe
-        n_format = NotifyFormat.TEXT.value
-    else:
-        n_format = NotifyFormat.TEXT.value
-
-    return n_format
-
-
-def apply_service_tweaks(url, n_body, n_title, requested_output_format):
-
-    # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
-    # Because different notifications may require different pre-processing, run each sequentially :(
-    # 2000 bytes minus -
-    #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
-    #     Length of URL - Incase they specify a longer custom avatar_url
-
-    if not n_body or not n_body.strip():
-        return url, n_body, n_title
-
-    # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
-    parsed = urlparse(url)
-    k = '?' if not parsed.query else '&'
-    if url and not 'avatar_url' in url \
-            and not url.startswith('mail') \
-            and not url.startswith('post') \
-            and not url.startswith('get') \
-            and not url.startswith('delete') \
-            and not url.startswith('put'):
-        url += k + f"avatar_url={APPRISE_AVATAR_URL}"
-
-    if url.startswith('tgram://'):
-        # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
-        # re https://github.com/dgtlmoon/changedetection.io/issues/555
-        # @todo re-use an existing library we have already imported to strip all non-allowed tags
-        n_body = n_body.replace('<br>', '\n')
-        n_body = n_body.replace('</br>', '\n')
-
-        # Use strikethrough for removed content, bold for added content
-        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '<s>')
-        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '</s>')
-        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '<b>')
-        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '</b>')
-        # Handle changed/replaced lines (old → new)
-        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, '<s>')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, '</s>')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, '<b>')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, '</b>')
-
-        # real limit is 4096, but minus some for extra metadata
-        payload_max_size = 3600
-        body_limit = max(0, payload_max_size - len(n_title))
-        n_title = n_title[0:payload_max_size]
-        n_body = n_body[0:body_limit]
-
-    elif (url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks')
-          or url.startswith('https://discord.com/api'))\
-            and 'html' in requested_output_format:
-        # Discord doesn't support HTML, replace <br> with newlines
-        n_body = n_body.strip().replace('<br>', '\n')
-        n_body = n_body.replace('</br>', '\n')
-
-        # Don't replace placeholders or truncate here - let the custom Discord plugin handle it
-        # The plugin will use embeds (6000 char limit across all embeds) if placeholders are present,
-        # or plain content (2000 char limit) otherwise
-
-        # Only do placeholder replacement if NOT using htmlcolor (which triggers embeds in custom plugin)
-        if requested_output_format == 'html':
-            # No diff placeholders, use Discord markdown for any other formatting
-            # Use Discord markdown: strikethrough for removed, bold for added
-            n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '~~')
-            n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '~~')
-            n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '**')
-            n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '**')
-            # Handle changed/replaced lines (old → new)
-            n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, '~~')
-            n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, '~~')
-            n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, '**')
-            n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, '**')
-
-            # Apply 2000 char limit for plain content
-            payload_max_size = 1700
-            body_limit = max(0, payload_max_size - len(n_title))
-            n_title = n_title[0:payload_max_size]
-            n_body = n_body[0:body_limit]
-        # else: our custom Discord plugin will convert any placeholders left over into embeds with color bars
-
-    # Is not discord/tgram and they want htmlcolor
-    elif requested_output_format == 'htmlcolor':
-        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, f'<span style="{HTML_REMOVED_STYLE}">')
-        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, f'</span>')
-        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, f'<span style="{HTML_ADDED_STYLE}">')
-        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, f'</span>')
-        # Handle changed/replaced lines (old → new)
-        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'<span style="{HTML_REMOVED_STYLE}">')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'</span>')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'<span style="{HTML_ADDED_STYLE}">')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'</span>')
-        n_body = n_body.replace("\n", '<br>')
-    elif requested_output_format == 'html':
-        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ')
-        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '')
-        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '(added) ')
-        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')
-        n_body = n_body.replace("\n", '<br>')
-
-    else: #plaintext etc default
-        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ')
-        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '')
-        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '(added) ')
-        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')
-
-    return url, n_body, n_title
-
-
-def process_notification(n_object: NotificationContextData, datastore):
-    from changedetectionio.jinja2_custom import render as jinja_render
+def process_notification(n_object, datastore):
+    from changedetectionio.safe_jinja import render as jinja_render
     from . import default_notification_format_for_watch, default_notification_format, valid_notification_formats
     # be sure its registered
     from .apprise_plugin.custom_handlers import apprise_http_custom_handler
-    # Register custom Discord plugin
-    from .apprise_plugin.discord import NotifyDiscordCustom
-
-    # Create list of custom handler protocols (both http and https versions)
-    custom_handler_protocols = [f"{method}://" for method in SUPPORTED_HTTP_METHODS]
-    custom_handler_protocols += [f"{method}s://" for method in SUPPORTED_HTTP_METHODS]
-
-    has_custom_handler = any(
-        url.startswith(tuple(custom_handler_protocols))
-        for url in n_object['notification_urls']
-    )
-
-    if not isinstance(n_object, NotificationContextData):
-        raise TypeError(f"Expected NotificationContextData, got {type(n_object)}")
 
     now = time.time()
     if n_object.get('notification_timestamp'):
         logger.trace(f"Time since queued {now-n_object['notification_timestamp']:.3f}s")
-
     # Insert variables into the notification content
     notification_parameters = create_notification_parameters(n_object, datastore)
 
-    requested_output_format = valid_notification_formats.get(
+    n_format = valid_notification_formats.get(
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
     )
 
     # If we arrived with 'System default' then look it up
-    if requested_output_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
+    if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
         # Initially text or whatever
-        requested_output_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format]).lower()
-
-    requested_output_format_original = requested_output_format
-
-    requested_output_format = notification_format_align_with_apprise(n_format=requested_output_format)
+        n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
 
     logger.trace(f"Complete notification body including Jinja and placeholders calculated in  {time.time() - now:.2f}s")
 
-    # If we have custom handlers, use invalid format to prevent conversion
-    # Otherwise use the proper format
-    if has_custom_handler:
-        input_format = 'raw-no-convert'
-
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
     # Anything higher than or equal to WARNING (which covers things like Connection errors)
     # raise it as an exception
@@ -249,51 +39,16 @@ def process_notification(n_object: NotificationContextData, datastore):
 
     apobj = apprise.Apprise(debug=True, asset=apprise_asset)
 
-    # Override Apprise's built-in Discord plugin with our custom one
-    # This allows us to use colored embeds for diff content
-    # First remove the built-in discord plugin, then add our custom one
-    apprise.plugins.N_MGR.remove('discord')
-    apprise.plugins.N_MGR.add(NotifyDiscordCustom, schemas='discord')
-
     if not n_object.get('notification_urls'):
         return None
 
     with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
         for url in n_object['notification_urls']:
-            parsed_url = urlparse(url)
-            prefix_add_to_url = '?' if not parsed_url.query else '&'
 
             # Get the notification body from datastore
             n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
-
-            if n_object.get('markup_text_to_html'):
-                n_body = markup_text_links_to_html(body=n_body)
-
-            # This actually means we request "Markdown to HTML"
-            if requested_output_format == NotifyFormat.MARKDOWN.value:
-                output_format = NotifyFormat.HTML.value
-                input_format = NotifyFormat.MARKDOWN.value
-                if not 'format=' in url.lower():
-                    url = f"{url}{prefix_add_to_url}format={output_format}"
-
-            # Deviation from apprise.
-            # No conversion, its like they want to send raw HTML but we add linebreaks
-            elif requested_output_format == NotifyFormat.HTML.value:
-                # same in and out means apprise wont try to convert
-                input_format = output_format = NotifyFormat.HTML.value
-                if not 'format=' in url.lower():
-                    url = f"{url}{prefix_add_to_url}format={output_format}"
-
-            else:
-                # Nothing to be done, leave it as plaintext
-                # `body_format` Tell apprise what format the INPUT is in
-                # &format= in URL Tell apprise what format the OUTPUT should be in (it can convert between)
-                input_format = output_format = NotifyFormat.TEXT.value
-                if not 'format=' in url.lower():
-                    url = f"{url}{prefix_add_to_url}format={output_format}"
-
-            if has_custom_handler:
-                input_format='raw-no-convert'
+            if n_object.get('notification_format', '').startswith('HTML'):
+                n_body = n_body.replace("\n", '<br>')
 
             n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
 
@@ -309,28 +64,74 @@ def process_notification(n_object: NotificationContextData, datastore):
             logger.info(f">> Process Notification: AppRise notifying {url}")
             url = jinja_render(template_str=url, **notification_parameters)
 
-            (url, n_body, n_title) = apply_service_tweaks(url=url, n_body=n_body, n_title=n_title, requested_output_format=requested_output_format_original)
+            # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
+            # Because different notifications may require different pre-processing, run each sequentially :(
+            # 2000 bytes minus -
+            #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
+            #     Length of URL - Incase they specify a longer custom avatar_url
+
+            # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
+            k = '?' if not '?' in url else '&'
+            if not 'avatar_url' in url \
+                    and not url.startswith('mail') \
+                    and not url.startswith('post') \
+                    and not url.startswith('get') \
+                    and not url.startswith('delete') \
+                    and not url.startswith('put'):
+                url += k + f"avatar_url={APPRISE_AVATAR_URL}"
+
+            if url.startswith('tgram://'):
+                # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
+                # re https://github.com/dgtlmoon/changedetection.io/issues/555
+                # @todo re-use an existing library we have already imported to strip all non-allowed tags
+                n_body = n_body.replace('<br>', '\n')
+                n_body = n_body.replace('</br>', '\n')
+                # real limit is 4096, but minus some for extra metadata
+                payload_max_size = 3600
+                body_limit = max(0, payload_max_size - len(n_title))
+                n_title = n_title[0:payload_max_size]
+                n_body = n_body[0:body_limit]
+
+            elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith(
+                    'https://discord.com/api'):
+                # real limit is 2000, but minus some for extra metadata
+                payload_max_size = 1700
+                body_limit = max(0, payload_max_size - len(n_title))
+                n_title = n_title[0:payload_max_size]
+                n_body = n_body[0:body_limit]
+
+            elif url.startswith('mailto'):
+                # Apprise will default to HTML, so we need to override it
+                # So that whats' generated in n_body is in line with what is going to be sent.
+                # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
+                if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
+                    prefix = '?' if not '?' in url else '&'
+                    # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
+                    n_format = n_format.lower()
+                    url = f"{url}{prefix}format={n_format}"
+                # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
 
             apobj.add(url)
 
             sent_objs.append({'title': n_title,
                               'body': n_body,
-                              'url': url})
+                              'url': url,
+                              'body_format': n_format})
 
+        # Blast off the notifications tht are set in .add()
         apobj.notify(
             title=n_title,
             body=n_body,
-            # `body_format` Tell apprise what format the INPUT is in
-            # &format= in URL Tell apprise what format the OUTPUT should be in (it can convert between)
-            body_format=input_format,
+            body_format=n_format,
             # False is not an option for AppRise, must be type None
             attach=n_object.get('screenshot', None)
         )
 
+
         # Returns empty string if nothing found, multi-line string otherwise
         log_value = logs.getvalue()
 
-        if log_value and ('WARNING' in log_value or 'ERROR' in log_value):
+        if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
             logger.critical(log_value)
             raise Exception(log_value)
 
@@ -340,15 +141,17 @@ def process_notification(n_object: NotificationContextData, datastore):
 
 # Notification title + body content parameters get created here.
 # ( Where we prepare the tokens in the notification to be replaced with actual values )
-def create_notification_parameters(n_object: NotificationContextData, datastore):
-    if not isinstance(n_object, NotificationContextData):
-        raise TypeError(f"Expected NotificationContextData, got {type(n_object)}")
+def create_notification_parameters(n_object, datastore):
+    from copy import deepcopy
+    from . import valid_tokens
 
-    watch = datastore.data['watching'].get(n_object['uuid'])
-    if watch:
-        watch_title = datastore.data['watching'][n_object['uuid']].label
+    # in the case we send a test notification from the main settings, there is no UUID.
+    uuid = n_object['uuid'] if 'uuid' in n_object else ''
+
+    if uuid:
+        watch_title = datastore.data['watching'][uuid].label
         tag_list = []
-        tags = datastore.get_all_tags_for_watch(n_object['uuid'])
+        tags = datastore.get_all_tags_for_watch(uuid)
         if tags:
             for tag_uuid, tag in tags.items():
                 tag_list.append(tag.get('title'))
@@ -363,10 +166,14 @@ def create_notification_parameters(n_object: NotificationContextData, datastore)
 
     watch_url = n_object['watch_url']
 
-    diff_url = "{}/diff/{}".format(base_url, n_object['uuid'])
-    preview_url = "{}/preview/{}".format(base_url, n_object['uuid'])
+    diff_url = "{}/diff/{}".format(base_url, uuid)
+    preview_url = "{}/preview/{}".format(base_url, uuid)
 
-    n_object.update(
+    # Not sure deepcopy is needed here, but why not
+    tokens = deepcopy(valid_tokens)
+
+    # Valid_tokens also used as a field validator
+    tokens.update(
         {
             'base_url': base_url,
             'diff_url': diff_url,
@@ -374,10 +181,13 @@ def create_notification_parameters(n_object: NotificationContextData, datastore)
             'watch_tag': watch_tag if watch_tag is not None else '',
             'watch_title': watch_title if watch_title is not None else '',
             'watch_url': watch_url,
-            'watch_uuid': n_object['uuid'],
+            'watch_uuid': uuid,
         })
 
-    if watch:
-        n_object.update(datastore.data['watching'].get(n_object['uuid']).extra_notification_token_values())
+    # n_object will contain diff, diff_added etc etc
+    tokens.update(n_object)
 
-    return n_object
+    if uuid:
+        tokens.update(datastore.data['watching'].get(uuid).extra_notification_token_values())
+
+    return tokens

changedetectionio/notification_service.py

@@ -6,51 +6,9 @@ Extracted from update_worker.py to provide standalone notification functionality
 for both sync and async workers
 """
 
-from loguru import logger
 import time
+from loguru import logger
 
-from changedetectionio.notification import default_notification_format
-
-# What is passed around as notification context, also used as the complete list of valid {{ tokens }}
-class NotificationContextData(dict):
-    def __init__(self, initial_data=None, **kwargs):
-        super().__init__({
-            'current_snapshot': None,
-            'diff': None,
-            'diff_added': None,
-            'diff_full': None,
-            'diff_patch': None,
-            'diff_removed': None,
-            'notification_timestamp': time.time(),
-            'screenshot': None,
-            'triggered_text': None,
-            'uuid': 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX',  # Converted to 'watch_uuid' in create_notification_parameters
-            'watch_url': 'https://WATCH-PLACE-HOLDER/',
-            'base_url': None,
-            'diff_url': None,
-            'preview_url': None,
-            'watch_tag': None,
-            'watch_title': None,
-            'markup_text_to_html': False, # If automatic conversion of plaintext to HTML should happen
-        })
-
-        # Apply any initial data passed in
-        self.update({'watch_uuid': self.get('uuid')})
-        if initial_data:
-            self.update(initial_data)
-
-        # Apply any keyword arguments
-        if kwargs:
-            self.update(kwargs)
-
-    def set_random_for_validation(self):
-        import random, string
-        """Randomly fills all dict keys with random strings (for validation/testing)."""
-        for key in self.keys():
-            if key in ['uuid', 'time', 'watch_uuid']:
-                continue
-            rand_str = 'RANDOM-PLACEHOLDER-'+''.join(random.choices(string.ascii_letters + string.digits, k=12))
-            self[key] = rand_str
 
 class NotificationService:
     """
@@ -62,16 +20,13 @@ class NotificationService:
         self.datastore = datastore
         self.notification_q = notification_q
     
-    def queue_notification_for_watch(self, n_object: NotificationContextData, watch):
+    def queue_notification_for_watch(self, n_object, watch):
         """
         Queue a notification for a watch with full diff rendering and template variables
         """
         from changedetectionio import diff
         from changedetectionio.notification import default_notification_format_for_watch
 
-        if not isinstance(n_object, NotificationContextData):
-            raise TypeError(f"Expected NotificationContextData, got {type(n_object)}")
-
         dates = []
         trigger_text = ''
 
@@ -92,6 +47,7 @@ class NotificationService:
         if n_object.get('notification_format') == default_notification_format_for_watch:
             n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')
 
+        html_colour_enable = False
         # HTML needs linebreak, but MarkDown and Text can use a linefeed
         if n_object.get('notification_format') == 'HTML':
             line_feed_sep = "<br>"
@@ -101,6 +57,7 @@ class NotificationService:
             line_feed_sep = "<br>"
             # Snapshot will be plaintext on the disk, convert to some kind of HTML
             snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
+            html_colour_enable = True
         else:
             line_feed_sep = "\n"
 
@@ -121,16 +78,16 @@ class NotificationService:
 
         n_object.update({
             'current_snapshot': snapshot_contents,
-            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep),
+            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
             'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
-            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep),
+            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
             'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
             'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
+            'notification_timestamp': now,
             'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
             'triggered_text': triggered_text,
             'uuid': watch.get('uuid') if watch else None,
             'watch_url': watch.get('url') if watch else None,
-            'watch_uuid': watch.get('uuid') if watch else None,
         })
 
         if watch:
@@ -183,7 +140,7 @@ class NotificationService:
         """
         Send notification when content changes are detected
         """
-        n_object = NotificationContextData()
+        n_object = {}
         watch = self.datastore.data['watching'].get(watch_uuid)
         if not watch:
             return
@@ -226,26 +183,11 @@ class NotificationService:
         if not watch:
             return
 
-        n_format = self.datastore.data['settings']['application'].get('notification_format', default_notification_format)
-        filter_list = ", ".join(watch['include_filters'])
-        # @todo - This could be a markdown template on the disk, apprise will convert the markdown to HTML+Plaintext parts in the email, and then 'markup_text_to_html' is not needed
-        body = f"""Hello,
-
-Your configured CSS/xPath filters of '{filter_list}' for {{{{watch_url}}}} did not appear on the page after {threshold} attempts.
-
-It's possible the page changed layout and the filter needs updating ( Try the 'Visual Selector' tab )
-
-Edit link: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}
-
-Thanks - Your omniscient changedetection.io installation.
-"""
-
-        n_object = NotificationContextData({
-            'notification_title': 'Changedetection.io - Alert - CSS/xPath filter was not present in the page',
-            'notification_body': body,
-            'notification_format': n_format,
-            'markup_text_to_html': n_format.lower().startswith('html')
-        })
+        n_object = {'notification_title': 'Changedetection.io - Alert - CSS/xPath filter was not present in the page',
+                    'notification_body': "Your configured CSS/xPath filters of '{}' for {{{{watch_url}}}} did not appear on the page after {} attempts, did the page change layout?\n\nLink: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\nThanks - Your omniscient changedetection.io installation :)\n".format(
+                        ", ".join(watch['include_filters']),
+                        threshold),
+                    'notification_format': 'text'}
 
         if len(watch['notification_urls']):
             n_object['notification_urls'] = watch['notification_urls']
@@ -273,28 +215,12 @@ Thanks - Your omniscient changedetection.io installation.
         if not watch:
             return
         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
-        n_format = self.datastore.data['settings']['application'].get('notification_format', default_notification_format).lower()
-        step = step_n + 1
-        # @todo - This could be a markdown template on the disk, apprise will convert the markdown to HTML+Plaintext parts in the email, and then 'markup_text_to_html' is not needed
-
-        # {{{{ }}}} because this will be Jinja2 {{ }} tokens
-        body = f"""Hello,
-        
-Your configured browser step at position {step} for the web page watch {{{{watch_url}}}} did not appear on the page after {threshold} attempts, did the page change layout?
-
-The element may have moved and needs editing, or does it need a delay added?
-
-Edit link: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}
-
-Thanks - Your omniscient changedetection.io installation.
-"""
-
-        n_object = NotificationContextData({
-            'notification_title': f"Changedetection.io - Alert - Browser step at position {step} could not be run",
-            'notification_body': body,
-            'notification_format': n_format,
-            'markup_text_to_html': n_format.lower().startswith('html')
-        })
+        n_object = {'notification_title': "Changedetection.io - Alert - Browser step at position {} could not be run".format(step_n+1),
+                    'notification_body': "Your configured browser step at position {} for {{{{watch_url}}}} "
+                                         "did not appear on the page after {} attempts, did the page change layout? "
+                                         "Does it need a delay added?\n\nLink: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\n"
+                                         "Thanks - Your omniscient changedetection.io installation :)\n".format(step_n+1, threshold),
+                    'notification_format': 'text'}
 
         if len(watch['notification_urls']):
             n_object['notification_urls'] = watch['notification_urls']

changedetectionio/processors/__init__.py

@@ -102,7 +102,7 @@ class difference_detection_processor():
             self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid'))
 
         # Tweak the base config with the per-watch ones
-        from changedetectionio.jinja2_custom import render as jinja_render
+        from changedetectionio.safe_jinja import render as jinja_render
         request_headers = CaseInsensitiveDict()
 
         ua = self.datastore.data['settings']['requests'].get('default_ua')

changedetectionio/processors/magic.py

@@ -64,31 +64,24 @@ class guess_stream_type():
         # Remove whitespace between < and tag name for robust detection (handles '< html', '<\nhtml', etc.)
         test_content_normalized = re.sub(r'<\s+', '<', test_content)
 
-        # Use puremagic for lightweight MIME detection (saves ~14MB vs python-magic)
+        # Magic will sometimes call text/plain as text/html!
         magic_result = None
         try:
-            import puremagic
+            import magic
 
-            # puremagic needs bytes, so encode if we have a string
-            content_bytes = content[:200].encode('utf-8') if isinstance(content, str) else content[:200]
-
-            # puremagic returns a list of PureMagic objects with confidence scores
-            detections = puremagic.magic_string(content_bytes)
-            if detections:
-                # Get the highest confidence detection
-                mime = detections[0].mime_type
-                logger.debug(f"Guessing mime type, original content_type '{http_content_header}', mime type detected '{mime}'")
-                if mime and "/" in mime:
-                    magic_result = mime
-                    # Ignore generic/fallback mime types
-                    if mime in ['application/octet-stream', 'application/x-empty', 'binary']:
-                        logger.debug(f"Ignoring generic mime type '{mime}' from puremagic library")
-                    # Trust puremagic for non-text types immediately
-                    elif mime not in ['text/html', 'text/plain']:
-                        magic_content_header = mime
+            mime = magic.from_buffer(content[:200], mime=True) # Send the original content
+            logger.debug(f"Guessing mime type, original content_type '{http_content_header}', mime type detected '{mime}'")
+            if mime and "/" in mime:
+                magic_result = mime
+                # Ignore generic/fallback mime types from magic
+                if mime in ['application/octet-stream', 'application/x-empty', 'binary']:
+                    logger.debug(f"Ignoring generic mime type '{mime}' from magic library")
+                # Trust magic for non-text types immediately
+                elif mime not in ['text/html', 'text/plain']:
+                    magic_content_header = mime
 
         except Exception as e:
-            logger.error(f"Error getting a more precise mime type from 'puremagic' library ({str(e)}), using content-based detection")
+            logger.error(f"Error getting a more precise mime type from 'magic' library ({str(e)}), using content-based detection")
 
         # Content-based detection (most reliable for text formats)
         # Check for HTML patterns first - if found, override magic's text/plain
@@ -101,21 +94,24 @@ class guess_stream_type():
             self.is_rss = True
         elif any(s in http_content_header for s in JSON_CONTENT_TYPES):
             self.is_json = True
-        elif 'pdf' in magic_content_header:
-            self.is_pdf = True
-        elif has_html_patterns or http_content_header == 'text/html':
-            self.is_html = True
-        elif any(s in magic_content_header for s in JSON_CONTENT_TYPES):
-            self.is_json = True
-        # magic will call a rss document 'xml'
-        # Rarely do endpoints give the right header, usually just text/xml, so we check also for <rss
-        # This also triggers the automatic CDATA text parser so the RSS goes back a nice content list
-        elif '<rss' in test_content_normalized or '<feed' in test_content_normalized or any(s in magic_content_header for s in RSS_XML_CONTENT_TYPES) or '<rdf:' in test_content_normalized:
-            self.is_rss = True
         elif any(s in http_content_header for s in XML_CONTENT_TYPES):
             # Only mark as generic XML if not already detected as RSS
             if not self.is_rss:
                 self.is_xml = True
+        elif 'pdf' in magic_content_header:
+            self.is_pdf = True
+###
+        elif has_html_patterns or http_content_header == 'text/html':
+            self.is_html = True
+        # If magic says text/plain and we found no HTML patterns, trust it
+        elif magic_result == 'text/plain':
+            self.is_plaintext = True
+            logger.debug(f"Trusting magic's text/plain result (no HTML patterns detected)")
+        elif any(s in magic_content_header for s in JSON_CONTENT_TYPES):
+            self.is_json = True
+        # magic will call a rss document 'xml'
+        elif '<rss' in test_content_normalized or '<feed' in test_content_normalized or any(s in magic_content_header for s in RSS_XML_CONTENT_TYPES):
+            self.is_rss = True
         elif test_content_normalized.startswith('<?xml') or any(s in magic_content_header for s in XML_CONTENT_TYPES):
             # Generic XML that's not RSS/Atom (RSS/Atom checked above)
             self.is_xml = True
@@ -126,8 +122,4 @@ class guess_stream_type():
         # Only trust magic for 'text' if no other patterns matched
         elif 'text' in magic_content_header:
             self.is_plaintext = True
-        # If magic says text/plain and we found no HTML patterns, trust it
-        elif magic_result == 'text/plain':
-            self.is_plaintext = True
-            logger.debug(f"Trusting magic's text/plain result (no HTML patterns detected)")
 

changedetectionio/processors/text_json_diff/processor.py

@@ -7,7 +7,6 @@ import re
 import urllib3
 
 from changedetectionio.conditions import execute_ruleset_against_all_plugins
-from changedetectionio.diff import ADDED_PLACEMARKER_OPEN
 from changedetectionio.processors import difference_detection_processor
 from changedetectionio.html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text, TRANSLATE_WHITESPACE_TABLE
 from changedetectionio import html_tools, content_fetchers
@@ -229,21 +228,8 @@ class ContentProcessor:
         self.datastore = datastore
 
     def preprocess_rss(self, content):
-        """
-        Convert CDATA/comments in RSS to usable text.
-
-        Supports two RSS processing modes:
-        - 'default': Inline CDATA replacement (original behavior)
-        - 'formatted': Format RSS items with title, link, guid, pubDate, and description (CDATA unmarked)
-        """
-        from changedetectionio import rss_tools
-        rss_mode = self.datastore.data["settings"]["application"].get("rss_reader_mode")
-        if rss_mode:
-            # Format RSS items nicely with CDATA content unmarked and converted to text
-            return rss_tools.format_rss_items(content)
-        else:
-            # Default: Original inline CDATA replacement
-            return cdata_in_document_to_text(html_content=content)
+        """Convert CDATA/comments in RSS to usable text."""
+        return cdata_in_document_to_text(html_content=content)
 
     def preprocess_pdf(self, raw_content):
         """Convert PDF to HTML using external tool."""
@@ -325,13 +311,13 @@ class ContentProcessor:
                     append_pretty_line_formatting=not self.watch.is_source_type_url
                 )
 
-        # Raise error if filter returned nothing
-        if not filtered_content.strip():
-            raise FilterNotFoundInResponse(
-                msg=self.filter_config.include_filters,
-                screenshot=self.fetcher.screenshot,
-                xpath_data=self.fetcher.xpath_data
-            )
+            # Raise error if filter returned nothing
+            if not filtered_content.strip():
+                raise FilterNotFoundInResponse(
+                    msg=self.filter_config.include_filters,
+                    screenshot=self.fetcher.screenshot,
+                    xpath_data=self.fetcher.xpath_data
+                )
 
         return filtered_content
 
@@ -398,11 +384,6 @@ class perform_site_check(difference_detection_processor):
         # RSS preprocessing
         if stream_content_type.is_rss:
             content = content_processor.preprocess_rss(content)
-            if self.datastore.data["settings"]["application"].get("rss_reader_mode"):
-                # Now just becomes regular HTML that can have xpath/CSS applied (first of the set etc)
-                stream_content_type.is_rss = False
-                stream_content_type.is_html = True
-                self.fetcher.content = content
 
         # PDF preprocessing
         if watch.is_pdf or stream_content_type.is_pdf:
@@ -557,20 +538,6 @@ class perform_site_check(difference_detection_processor):
             else:
                 logger.debug(f"check_unique_lines: UUID {watch.get('uuid')} had unique content")
 
-        # Note: Explicit cleanup is only needed here because text_json_diff handles
-        # large strings (100KB-300KB for RSS/HTML). The other processors work with
-        # small strings and don't need this.
-        #
-        # Python would clean these up automatically, but explicit `del` frees memory
-        # immediately rather than waiting for function return, reducing peak memory usage.
-        del content
-        if 'html_content' in locals() and html_content is not stripped_text:
-            del html_content
-        if 'text_content_before_ignored_filter' in locals() and text_content_before_ignored_filter is not stripped_text:
-            del text_content_before_ignored_filter
-        if 'text_for_checksuming' in locals() and text_for_checksuming is not stripped_text:
-            del text_for_checksuming
-
         return changed_detected, update_obj, stripped_text
 
     def _apply_diff_filtering(self, watch, stripped_text, text_before_filter):

changedetectionio/rss_tools.py

@@ -1,130 +0,0 @@
-"""
-RSS/Atom feed processing tools for changedetection.io
-"""
-
-from loguru import logger
-import re
-
-
-def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False) -> str:
-    """
-    Process CDATA sections in HTML/XML content - inline replacement.
-
-    Args:
-        html_content: The HTML/XML content to process
-        render_anchor_tag_content: Whether to render anchor tag content
-
-    Returns:
-        Processed HTML/XML content with CDATA sections replaced inline
-    """
-    from xml.sax.saxutils import escape as xml_escape
-    from .html_tools import html_to_text
-
-    pattern = '<!\[CDATA\[(\s*(?:.(?<!\]\]>)\s*)*)\]\]>'
-
-    def repl(m):
-        text = m.group(1)
-        return xml_escape(html_to_text(html_content=text, render_anchor_tag_content=render_anchor_tag_content)).strip()
-
-    return re.sub(pattern, repl, html_content)
-
-
-def format_rss_items(rss_content: str, render_anchor_tag_content=False) -> str:
-    """
-    Format RSS/Atom feed items in a readable text format using feedparser.
-
-    Converts RSS <item> or Atom <entry> elements to formatted text with:
-    - <title> → <h1>Title</h1>
-    - <link> → Link: [url]
-    - <guid> → Guid: [id]
-    - <pubDate> → PubDate: [date]
-    - <description> or <content> → Raw HTML content (CDATA and entities automatically handled)
-
-    Args:
-        rss_content: The RSS/Atom feed content
-        render_anchor_tag_content: Whether to render anchor tag content in descriptions (unused, kept for compatibility)
-
-    Returns:
-        Formatted HTML content ready for html_to_text conversion
-    """
-    try:
-        import feedparser
-        from xml.sax.saxutils import escape as xml_escape
-
-        # Parse the feed - feedparser handles all RSS/Atom variants, CDATA, entity unescaping, etc.
-        feed = feedparser.parse(rss_content)
-
-        formatted_items = []
-
-        # Determine feed type for appropriate labels when fields are missing
-        # feedparser sets feed.version to things like 'rss20', 'atom10', etc.
-        is_atom = feed.version and 'atom' in feed.version
-
-        for entry in feed.entries:
-            item_parts = []
-
-            # Title - feedparser handles CDATA and entity unescaping automatically
-            if hasattr(entry, 'title') and entry.title:
-                item_parts.append(f'<h1>{xml_escape(entry.title)}</h1>')
-
-            # Link
-            if hasattr(entry, 'link') and entry.link:
-                item_parts.append(f'Link: {xml_escape(entry.link)}<br>')
-
-            # GUID/ID
-            if hasattr(entry, 'id') and entry.id:
-                item_parts.append(f'Guid: {xml_escape(entry.id)}<br>')
-
-            # Date - feedparser normalizes all date field names to 'published'
-            if hasattr(entry, 'published') and entry.published:
-                item_parts.append(f'PubDate: {xml_escape(entry.published)}<br>')
-
-            # Description/Content - feedparser handles CDATA and entity unescaping automatically
-            # Only add "Summary:" label for Atom <summary> tags
-            content = None
-            add_label = False
-
-            if hasattr(entry, 'content') and entry.content:
-                # Atom <content> - no label, just content
-                content = entry.content[0].value if entry.content[0].value else None
-            elif hasattr(entry, 'summary'):
-                # Could be RSS <description> or Atom <summary>
-                # feedparser maps both to entry.summary
-                content = entry.summary if entry.summary else None
-                # Only add "Summary:" label for Atom feeds (which use <summary> tag)
-                if is_atom:
-                    add_label = True
-
-            # Add content with or without label
-            if content:
-                if add_label:
-                    item_parts.append(f'Summary:<br>{content}')
-                else:
-                    item_parts.append(content)
-            else:
-                # No content - just show <none>
-                item_parts.append('&lt;none&gt;')
-
-            # Join all parts of this item
-            if item_parts:
-                formatted_items.append('\n'.join(item_parts))
-
-        # Wrap each item in a div with classes (first, last, item-N)
-        items_html = []
-        total_items = len(formatted_items)
-        for idx, item in enumerate(formatted_items):
-            classes = ['rss-item']
-            if idx == 0:
-                classes.append('first')
-            if idx == total_items - 1:
-                classes.append('last')
-            classes.append(f'item-{idx + 1}')
-
-            class_str = ' '.join(classes)
-            items_html.append(f'<div class="{class_str}">{item}</div>')
-        return '<html><body>\n'+"\n<br><br>".join(items_html)+'\n</body></html>'
-
-    except Exception as e:
-        logger.warning(f"Error formatting RSS items: {str(e)}")
-        # Fall back to original content
-        return rss_content

changedetectionio/run_basic_tests.sh

@@ -15,7 +15,7 @@ find tests/test_*py -type f|while read test_name
 do
   echo "TEST RUNNING $test_name"
   # REMOVE_REQUESTS_OLD_SCREENSHOTS disabled so that we can write a screenshot and send it in test_notifications.py without a real browser
-  REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv -s --maxfail=1 --tb=long $test_name
+  REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest $test_name
 done
 
 echo "RUNNING WITH BASE_URL SET"
@@ -23,20 +23,20 @@ echo "RUNNING WITH BASE_URL SET"
 # Now re-run some tests with BASE_URL enabled
 # Re #65 - Ability to include a link back to the installation, in the notification.
 export BASE_URL="https://really-unique-domain.io"
-REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv -s --maxfail=1 tests/test_notification.py
+REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest tests/test_notification.py
 
 
 # Re-run with HIDE_REFERER set - could affect login
 export HIDE_REFERER=True
-pytest -vv -s --maxfail=1 tests/test_access_control.py
+pytest tests/test_access_control.py
 
 # Re-run a few tests that will trigger brotli based storage
 export SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD=5
-pytest -vv -s --maxfail=1 tests/test_access_control.py
+pytest tests/test_access_control.py
 REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest tests/test_notification.py
-pytest -vv -s --maxfail=1 tests/test_backend.py
-pytest -vv -s --maxfail=1 tests/test_rss.py
-pytest -vv -s --maxfail=1 tests/test_unique_lines.py
+pytest tests/test_backend.py
+pytest tests/test_rss.py
+pytest tests/test_unique_lines.py
 
 # Try high concurrency
 FETCH_WORKERS=130 pytest  tests/test_history_consistency.py -v -l

changedetectionio/safe_jinja.py

@@ -0,0 +1,24 @@
+"""
+Safe Jinja2 render with max payload sizes
+
+See https://jinja.palletsprojects.com/en/3.1.x/sandbox/#security-considerations
+"""
+
+import jinja2.sandbox
+import typing as t
+import os
+
+JINJA2_MAX_RETURN_PAYLOAD_SIZE = 1024 * int(os.getenv("JINJA2_MAX_RETURN_PAYLOAD_SIZE_KB", 1024 * 10))
+
+# This is used for notifications etc, so actually it's OK to send custom HTML such as <a href> etc, but it should limit what data is available.
+# (Which also limits available functions that could be called)
+def render(template_str, **args: t.Any) -> str:
+    jinja2_env = jinja2.sandbox.ImmutableSandboxedEnvironment(extensions=['jinja2_time.TimeExtension'])
+    output = jinja2_env.from_string(template_str).render(args)
+    return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]
+
+def render_fully_escaped(content):
+    env = jinja2.sandbox.ImmutableSandboxedEnvironment(autoescape=True)
+    template = env.from_string("{{ some_html|e }}")
+    return template.render(some_html=content)
+

changedetectionio/static/js/global-settings.js

@@ -29,7 +29,7 @@ $(document).ready(function () {
         $(this).text(new Date($(this).data("utc")).toLocaleString());
     })
 
-    const timezoneInput = $('#application-scheduler_timezone_default');
+    const timezoneInput = $('#application-timezone');
     if(timezoneInput.length) {
         const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
         if (!timezoneInput.val().trim()) {

changedetectionio/static/js/realtime.js

@@ -2,13 +2,6 @@
 
 $(document).ready(function () {
 
-    function reapplyTableStripes() {
-        $('.watch-table tbody tr').each(function(index) {
-            $(this).removeClass('pure-table-odd pure-table-even');
-            $(this).addClass(index % 2 === 0 ? 'pure-table-odd' : 'pure-table-even');
-        });
-    }
-
     function bindSocketHandlerButtonsEvents(socket) {
         $('.ajax-op').on('click.socketHandlerNamespace', function (e) {
             e.preventDefault();
@@ -108,7 +101,6 @@ $(document).ready(function () {
             socket.on('watch_deleted', function (data) {
                 $('tr[data-watch-uuid="' + data.uuid + '"] td').fadeOut(500, function () {
                     $(this).closest('tr').remove();
-                    reapplyTableStripes();
                 });
             });
 

changedetectionio/static/styles/scss/styles.scss

@@ -344,7 +344,7 @@ label {
  }  
 }
 
-.grey-form-border {
+#notification-customisation {
   border: 1px solid var(--color-border-notification);
   padding: 0.5rem;
   border-radius: 5px;

changedetectionio/store.py

@@ -228,36 +228,26 @@ class ChangeDetectionStore:
         d['settings']['application']['active_base_url'] = active_base_url.strip('" ')
         return d
 
-    from pathlib import Path
-
-    def delete_path(self, path: Path):
-        import shutil
-        """Delete a file or directory tree, including the path itself."""
-        if not path.exists():
-            return
-        if path.is_file() or path.is_symlink():
-            path.unlink(missing_ok=True)  # deletes a file or symlink
-        else:
-            shutil.rmtree(path, ignore_errors=True)  # deletes dir *and* its contents
-
     # Delete a single watch by UUID
     def delete(self, uuid):
         import pathlib
+        import shutil
 
         with self.lock:
             if uuid == 'all':
                 self.__data['watching'] = {}
                 time.sleep(1) # Mainly used for testing to allow all items to flush before running next test
+
+                # GitHub #30 also delete history records
                 for uuid in self.data['watching']:
                     path = pathlib.Path(os.path.join(self.datastore_path, uuid))
                     if os.path.exists(path):
-                        self.delete(uuid)
+                        shutil.rmtree(path)
 
             else:
                 path = pathlib.Path(os.path.join(self.datastore_path, uuid))
                 if os.path.exists(path):
-                    self.delete_path(path)
-
+                    shutil.rmtree(path)
                 del self.data['watching'][uuid]
 
         self.needs_write_urgent = True
@@ -986,10 +976,6 @@ class ChangeDetectionStore:
         if self.data['settings']['application'].get('extract_title_as_title'):
             self.data['settings']['application']['ui']['use_page_title_in_list'] = self.data['settings']['application'].get('extract_title_as_title')
 
-    def update_21(self):
-        self.data['settings']['application']['scheduler_timezone_default'] = self.data['settings']['application'].get('timezone')
-        del self.data['settings']['application']['timezone']
-
 
     def add_notification_url(self, notification_url):
         

changedetectionio/templates/_common_fields.html

@@ -33,7 +33,7 @@
                                 <div id="notification-test-log" style="display: none;"><span class="pure-form-message-inline">Processing..</span></div>
                             </div>
                         </div>
-                        <div class="pure-control-group grey-form-border">
+                        <div id="notification-customisation" class="pure-control-group">
                             <div class="pure-control-group">
                                 {{ render_field(form.notification_title, class="m-d notification-title", placeholder=settings_application['notification_title']) }}
                                 <span class="pure-form-message-inline">Title for all notifications</span>
@@ -134,12 +134,6 @@
                                     <p>
                                         URL encoding, use <strong>|urlencode</strong>, for example - <code>gets://hook-website.com/test.php?title={{ '{{ watch_title|urlencode }}' }}</code>
                                     </p>
-                                    <p>
-                                        Regular-expression replace, use <strong>|regex_replace</strong>, for example -   <code>{{ "{{ \"hello world 123\" | regex_replace('[0-9]+', 'no-more-numbers') }}" }}</code>
-                                    </p>
-                                    <p>
-                                        For a complete reference of all Jinja2 built-in filters, users can refer to the <a href="https://jinja.palletsprojects.com/en/3.1.x/templates/#builtin-filters">https://jinja.palletsprojects.com/en/3.1.x/templates/#builtin-filters</a>
-                                    </p>
                                 </div>
                             </div>
                             <div class="pure-control-group">

changedetectionio/templates/_helpers.html

@@ -14,31 +14,13 @@
                 {% if field.errors is mapping and 'form' in field.errors %}
                     {#  and subfield form errors, such as used in RequiredFormField() for TimeBetweenCheckForm sub form #}
                     {% set errors = field.errors['form'] %}
-                    {% for error in errors %}
-                        <li>{{ error }}</li>
-                    {% endfor %}
-                {% elif field.type == 'FieldList' %}
-                    {# Handle FieldList of FormFields - errors is a list of dicts, one per entry #}
-                    {% for idx, entry_errors in field.errors|enumerate %}
-                        {% if entry_errors is mapping and entry_errors %}
-                            {# Only show entries that have actual errors #}
-                            <li><strong>Entry {{ idx + 1 }}:</strong>
-                                <ul>
-                                    {% for field_name, messages in entry_errors.items() %}
-                                        {% for message in messages %}
-                                            <li>{{ field_name }}: {{ message }}</li>
-                                        {% endfor %}
-                                    {% endfor %}
-                                </ul>
-                            </li>
-                        {% endif %}
-                    {% endfor %}
                 {% else %}
                     {#  regular list of errors with this field #}
-                    {% for error in field.errors %}
-                        <li>{{ error }}</li>
-                    {% endfor %}
+                    {% set errors = field.errors %}
                 {% endif %}
+                {% for error in errors %}
+                    <li>{{ error }}</li>
+                {% endfor %}
             </ul>
         {% endif %}
     </div>
@@ -111,39 +93,6 @@
   {{ field(**kwargs)|safe }}
 {% endmacro %}
 
-{% macro render_fieldlist_with_inline_errors(fieldlist) %}
-  {# Specialized macro for FieldList(FormField(...)) that renders errors inline with each field #}
-  <div {% if fieldlist.errors %} class="error" {% endif %}>{{ fieldlist.label }}</div>
-  <div {% if fieldlist.errors %} class="error" {% endif %}>
-    <ul id="{{ fieldlist.id }}">
-      {% for entry in fieldlist %}
-        <li {% if entry.errors %} class="error" {% endif %}>
-          <label for="{{ entry.id }}" {% if entry.errors %} class="error" {% endif %}>{{ fieldlist.label.text }}-{{ loop.index0 }}</label>
-          <table id="{{ entry.id }}" {% if entry.errors %} class="error" {% endif %}>
-            <tbody>
-              {% for subfield in entry %}
-                <tr {% if subfield.errors %} class="error" {% endif %}>
-                  <th {% if subfield.errors %} class="error" {% endif %}><label for="{{ subfield.id }}" {% if subfield.errors %} class="error" {% endif %}>{{ subfield.label.text }}</label></th>
-                  <td {% if subfield.errors %} class="error" {% endif %}>
-                    {{ subfield(**kwargs)|safe }}
-                    {% if subfield.errors %}
-                      <ul class="errors">
-                        {% for error in subfield.errors %}
-                          <li class="error">{{ error }}</li>
-                        {% endfor %}
-                      </ul>
-                    {% endif %}
-                  </td>
-                </tr>
-              {% endfor %}
-            </tbody>
-          </table>
-        </li>
-      {% endfor %}
-    </ul>
-  </div>
-{% endmacro %}
-
 {% macro render_conditions_fieldlist_of_formfields_as_table(fieldlist, table_id="rulesTable") %}
   <div class="fieldlist_formfields" id="{{ table_id }}">
     <div class="fieldlist-header">

changedetectionio/tests/conftest.py

@@ -4,7 +4,6 @@ import time
 from threading import Thread
 
 import pytest
-import arrow
 from changedetectionio import changedetection_app
 from changedetectionio import store
 import os
@@ -30,17 +29,6 @@ def reportlog(pytestconfig):
     logger.remove(handler_id)
 
 
-@pytest.fixture
-def environment(mocker):
-    """Mock arrow.now() to return a fixed datetime for testing jinja2 time extension."""
-    # Fixed datetime: Wed, 09 Dec 2015 23:33:01 UTC
-    # This is calculated to match the test expectations when offsets are applied
-    fixed_datetime = arrow.Arrow(2015, 12, 9, 23, 33, 1, tzinfo='UTC')
-    # Patch arrow.now in the TimeExtension module where it's actually used
-    mocker.patch('changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now', return_value=fixed_datetime)
-    return fixed_datetime
-
-
 def format_memory_human(bytes_value):
     """Format memory in human-readable units (KB, MB, GB)"""
     if bytes_value < 1024:

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -49,39 +49,3 @@ def test_select_custom(client, live_server, measure_memory_usage):
     #
     # Now we should see the request in the container logs for "squid-squid-custom" because it will be the only default
 
-
-def test_custom_proxy_validation(client, live_server, measure_memory_usage):
-    #  live_server_setup(live_server) # Setup on conftest per function
-
-    # Goto settings, add our custom one
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={
-            "requests-time_between_check-minutes": 180,
-            "application-ignore_whitespace": "y",
-            "application-fetch_backend": 'html_requests',
-            "requests-extra_proxies-0-proxy_name": "custom-test-proxy",
-            "requests-extra_proxies-0-proxy_url": "xxxxhtt/333??p://test:awesome@squid-custom:3128",
-        },
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." not in res.data
-    assert b'Proxy URLs must start with' in res.data
-
-
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={
-            "requests-time_between_check-minutes": 180,
-            "application-ignore_whitespace": "y",
-            "application-fetch_backend": 'html_requests',
-            "requests-extra_proxies-0-proxy_name": "custom-test-proxy",
-            "requests-extra_proxies-0-proxy_url": "https://",
-        },
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." not in res.data
-    assert b"Invalid URL." in res.data
-    

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -1,15 +1,14 @@
+import json
+import os
 import time
+import re
 from flask import url_for
-from email import message_from_string
-from email.policy import default as email_policy
-
-from changedetectionio.diff import HTML_REMOVED_STYLE, HTML_ADDED_STYLE
 from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
     wait_for_all_checks, \
     set_longer_modified_response, delete_all_watches
-
+from changedetectionio.tests.util import extract_UUID_from_client
 import logging
-
+import base64
 
 # NOTE - RELIES ON mailserver as hostname running, see github build recipes
 smtp_test_server = 'mailserver'
@@ -51,7 +50,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
         url_for("settings.settings_page"),
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": "some text\nfallback-body<br> " + default_notification_body,
+              "application-notification_body": "fallback-body<br> " + default_notification_body,
               "application-notification_format": 'HTML',
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
@@ -78,229 +77,19 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
 
     time.sleep(3)
 
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
 
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
-
-    # The email should have two bodies (multipart/alternative with text/plain and text/html)
-    assert msg.is_multipart()
-    assert msg.get_content_type() == 'multipart/alternative'
-
-    # Get the parts
-    parts = list(msg.iter_parts())
-    assert len(parts) == 2
-
-    # First part should be text/plain (the auto-generated plaintext version)
-    text_part = parts[0]
-    assert text_part.get_content_type() == 'text/plain'
-    text_content = text_part.get_content()
-    assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
-    assert 'fallback-body\r\n' in text_content  # The plaintext part
-
-    # Second part should be text/html
-    html_part = parts[1]
-    assert html_part.get_content_type() == 'text/html'
-    html_content = html_part.get_content()
-    assert 'some text<br>' in html_content  # We converted \n from the notification body
-    assert 'fallback-body<br>' in html_content  # kept the original <br>
-    assert '(added) So let\'s see what happens.<br>' in html_content  # the html part
-    delete_all_watches(client)
-
-
-def test_check_notification_plaintext_format(client, live_server, measure_memory_usage):
-    set_original_response()
-
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": "some text\n" + default_notification_body,
-              "application-notification_format": 'Plain Text',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    # Add a watch and trigger a HTTP POST
-    test_url = url_for('test_endpoint', _external=True)
-    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
-
-    set_longer_modified_response()
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
-
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
-
-    # The email should be plain text only (not multipart)
-    assert not msg.is_multipart()
-    assert msg.get_content_type() == 'text/plain'
-
-    # Get the plain text content
-    text_content = msg.get_content()
-    assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
-
-    # Should NOT contain HTML
-    assert '<br>' not in text_content  # We should not have HTML in plain text
-    delete_all_watches(client)
-
-
-
-def test_check_notification_html_color_format(client, live_server, measure_memory_usage):
-    set_original_response()
-
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": "some text\n" + default_notification_body,
-              "application-notification_format": 'HTML Color',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    # Add a watch and trigger a HTTP POST
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("ui.ui_views.form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    wait_for_all_checks(client)
-    set_longer_modified_response()
-    time.sleep(2)
-
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
-
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
-
-    # The email should have two bodies (multipart/alternative with text/plain and text/html)
-    assert msg.is_multipart()
-    assert msg.get_content_type() == 'multipart/alternative'
-
-    # Get the parts
-    parts = list(msg.iter_parts())
-    assert len(parts) == 2
-
-    # First part should be text/plain (the auto-generated plaintext version)
-    text_part = parts[0]
-    assert text_part.get_content_type() == 'text/plain'
-    text_content = text_part.get_content()
-    assert 'So let\'s see what happens.\r\n' in text_content  # The plaintext part
-    assert '(added)' not in text_content # Because apprise only dumb converts the html to text
-
-    # Second part should be text/html with color styling
-    html_part = parts[1]
-    assert html_part.get_content_type() == 'text/html'
-    html_content = html_part.get_content()
-    assert HTML_REMOVED_STYLE in html_content
-    assert HTML_ADDED_STYLE in html_content
-
-    assert 'some text<br>' in html_content
-    delete_all_watches(client)
-
-def test_check_notification_markdown_format(client, live_server, measure_memory_usage):
-    set_original_response()
-
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": "*header*\n\nsome text\n" + default_notification_body,
-              "application-notification_format": 'Markdown to HTML',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    # Add a watch and trigger a HTTP POST
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("ui.ui_views.form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    wait_for_all_checks(client)
-    set_longer_modified_response()
-    time.sleep(2)
-
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
-
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
-
-    # The email should have two bodies (multipart/alternative with text/plain and text/html)
-    assert msg.is_multipart()
-    assert msg.get_content_type() == 'multipart/alternative'
-
-    # Get the parts
-    parts = list(msg.iter_parts())
-    assert len(parts) == 2
-
-    # First part should be text/plain (the auto-generated plaintext version)
-    text_part = parts[0]
-    assert text_part.get_content_type() == 'text/plain'
-    text_content = text_part.get_content()
-    assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
-
-
-    # Second part should be text/html and roughly converted from markdown to HTML
-    html_part = parts[1]
-    assert html_part.get_content_type() == 'text/html'
-    html_content = html_part.get_content()
-    assert '<p><em>header</em></p>' in html_content
-    assert '(added) So let\'s see what happens.<br' in html_content
+    # The email should have two bodies, and the text/html part should be <br>
+    assert 'Content-Type: text/plain' in msg
+    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
+    assert 'Content-Type: text/html' in msg
+    assert '(added) So let\'s see what happens.<br>' in msg  # the html part
     delete_all_watches(client)
 
 
 def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage):
+    ##  live_server_setup(live_server) # Setup on conftest per function
 
     # HTML problems? see this
     # https://github.com/caronc/apprise/issues/633
@@ -326,7 +115,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
               "application-notification_body": notification_body,
-              "application-notification_format": 'Plain Text',
+              "application-notification_format": 'Text',
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
@@ -350,21 +139,15 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     wait_for_all_checks(client)
 
     time.sleep(3)
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
     #    with open('/tmp/m.txt', 'w') as f:
-    #        f.write(msg_raw)
-
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
+    #        f.write(msg)
 
     # The email should not have two bodies, should be TEXT only
-    assert not msg.is_multipart()
-    assert msg.get_content_type() == 'text/plain'
 
-    # Get the plain text content
-    text_content = msg.get_content()
-    assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
+    assert 'Content-Type: text/plain' in msg
+    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
 
     set_original_response()
     # Now override as HTML format
@@ -381,34 +164,18 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     wait_for_all_checks(client)
 
     time.sleep(3)
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
 
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
-
-    # The email should have two bodies (multipart/alternative)
-    assert msg.is_multipart()
-    assert msg.get_content_type() == 'multipart/alternative'
-
-    # Get the parts
-    parts = list(msg.iter_parts())
-    assert len(parts) == 2
-
-    # First part should be text/plain
-    text_part = parts[0]
-    assert text_part.get_content_type() == 'text/plain'
-    text_content = text_part.get_content()
-    assert '(removed) So let\'s see what happens.\r\n' in text_content  # The plaintext part
-
-    # Second part should be text/html
-    html_part = parts[1]
-    assert html_part.get_content_type() == 'text/html'
-    html_content = html_part.get_content()
-    assert '(removed) So let\'s see what happens.<br>' in html_content  # the html part
+    # The email should have two bodies, and the text/html part should be <br>
+    assert 'Content-Type: text/plain' in msg
+    assert '(removed) So let\'s see what happens.\r\n' in msg  # The plaintext part with \n
+    assert 'Content-Type: text/html' in msg
+    assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
 
     # https://github.com/dgtlmoon/changedetection.io/issues/2103
-    assert '<h1>Test</h1>' in html_content
-    assert '&lt;' not in html_content
+    assert '<h1>Test</h1>' in msg
+    assert '&lt;' not in msg
+    assert 'Content-Type: text/html' in msg
 
     delete_all_watches(client)

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -6,9 +6,6 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output, delete_all_watches
 import time
 
-from ..diff import ADDED_PLACEMARKER_OPEN
-
-
 def set_original(excluding=None, add_line=None):
     test_return_data = """<html>
      <body>
@@ -124,7 +121,6 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
               "application-notification_body": 'triggered text was -{{triggered_text}}- ### 网站监测 内容更新了 ####',
               # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
               "application-notification_urls": test_notification_url,
-              "application-notification_format": 'Plain Text',
               "application-minutes_between_check": 180,
               "application-fetch_backend": "html_requests"
               },
@@ -178,7 +174,6 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     assert os.path.isfile("test-datastore/notification.txt"), "Notification fired because I can see the output file"
     with open("test-datastore/notification.txt", 'rb') as f:
         response = f.read()
-        assert ADDED_PLACEMARKER_OPEN.encode('utf-8') not in response #  _apply_diff_filtering shouldnt add something here
         assert b'-Oh yes please' in response
         assert '网站监测 内容更新了'.encode('utf-8') in response
 

changedetectionio/tests/test_backend.py

@@ -165,46 +165,6 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # Cleanup everything
     delete_all_watches(client)
 
-
-# Server says its plaintext, we should always treat it as plaintext, and then if they have a filter, try to apply that
-def test_requests_timeout(client, live_server, measure_memory_usage):
-    delay = 2
-    test_url = url_for('test_endpoint', delay=delay, _external=True)
-
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-ui-use_page_title_in_list": "",
-              "requests-time_between_check-minutes": 180,
-              "requests-timeout": delay - 1,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    # Add our URL to the import page
-    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    # requests takes >2 sec but we timeout at 1 second
-    res = client.get(url_for("watchlist.index"))
-    assert b'Read timed out. (read timeout=1)' in res.data
-
-    ##### Now set a longer timeout
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-ui-use_page_title_in_list": "",
-              "requests-time_between_check-minutes": 180,
-              "requests-timeout": delay + 1, # timeout should be a second more than the reply time
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-
-    wait_for_all_checks(client)
-
-    res = client.get(url_for("watchlist.index"))
-    assert b'Read timed out' not in res.data
-
 def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage):
     """
 
@@ -378,4 +338,4 @@ def test_plaintext_even_if_xml_content_and_can_apply_filters(client, live_server
     assert b'<string name="feed_update_receiver_name"' in res.data
     assert b'<foobar' not in res.data
 
-    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_filter_exist_changes.py

@@ -86,16 +86,14 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
                                                    "Diff Full: {{diff_full}}\n"
                                                    "Diff as Patch: {{diff_patch}}\n"
                                                    ":-)",
-                              "notification_format": 'Plain Text'}
+                              "notification_format": "Text"}
 
     notification_form_data.update({
         "url": test_url,
         "tags": "my tag",
         "title": "my title",
         "headers": "",
-        # preprended with extra filter that intentionally doesn't match any entry,
-        # notification should still be sent even if first filter does not match (PR#3516)
-        "include_filters": ".non-matching-selector\n.ticket-available",
+        "include_filters": '.ticket-available',
         "fetch_backend": "html_requests",
         "time_between_check_use_default": "y"})
 

changedetectionio/tests/test_filter_failure_notification.py

@@ -1,8 +1,10 @@
 import os
 import time
+from loguru import logger
 from flask import url_for
-from .util import set_original_response,  wait_for_all_checks, wait_for_notification_endpoint_output
-from ..notification import valid_notification_formats
+from .util import set_original_response, live_server_setup, extract_UUID_from_client, wait_for_all_checks, \
+    wait_for_notification_endpoint_output
+from changedetectionio.model import App
 
 
 def set_response_with_filter():
@@ -21,14 +23,13 @@ def set_response_with_filter():
         f.write(test_return_data)
     return None
 
-def run_filter_test(client, live_server, content_filter, app_notification_format):
+def run_filter_test(client, live_server, content_filter):
 
     # Response WITHOUT the filter ID element
     set_original_response()
-    live_server.app.config['DATASTORE'].data['settings']['application']['notification_format'] = app_notification_format
 
     # Goto the edit page, add our ignore text
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'post')
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -63,7 +64,7 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
                                        "Diff Full: {{diff_full}}\n"
                                        "Diff as Patch: {{diff_patch}}\n"
                                        ":-)",
-                  "notification_format": 'Plain Text',
+                  "notification_format": "Text",
                   "fetch_backend": "html_requests",
                   "filter_failure_notification_send": 'y',
                   "time_between_check_use_default": "y",
@@ -126,23 +127,8 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
     with open("test-datastore/notification.txt", 'r') as f:
         notification = f.read()
 
-    assert 'Your configured CSS/xPath filters' in notification
-
-
-    # Text (or HTML conversion) markup to make the notifications a little nicer should have worked
-    if app_notification_format.startswith('html'):
-        # apprise should have used sax-escape (' instead of ", " etc), lets check it worked
-
-        from apprise.conversion import convert_between
-        from apprise.common import NotifyFormat
-        escaped_filter = convert_between(NotifyFormat.TEXT, NotifyFormat.HTML, content_filter)
-
-        assert escaped_filter in notification or escaped_filter.replace('"', '"') in notification
-        assert 'a href="' in notification # Quotes should still be there so the link works
-
-    else:
-        assert 'a href' not in notification
-        assert content_filter in notification
+    assert 'CSS/xPath filter was not present in the page' in notification
+    assert content_filter.replace('"', '\\"') in notification
 
     # Remove it and prove that it doesn't trigger when not expected
     # It should register a change, but no 'filter not found'
@@ -173,20 +159,14 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
     os.unlink("test-datastore/notification.txt")
 
 
+
+
 def test_check_include_filters_failure_notification(client, live_server, measure_memory_usage):
-    #   #  live_server_setup(live_server) # Setup on conftest per function
-    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('HTML Color'))
-    # Check markup send conversion didnt affect plaintext preference
-    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('Plain Text'))
+#   #  live_server_setup(live_server) # Setup on conftest per function
+    run_filter_test(client, live_server,'#nope-doesnt-exist')
 
 def test_check_xpath_filter_failure_notification(client, live_server, measure_memory_usage):
-    #   #  live_server_setup(live_server) # Setup on conftest per function
-    run_filter_test(client=client, live_server=live_server, content_filter='//*[@id="nope-doesnt-exist"]', app_notification_format=valid_notification_formats.get('HTML Color'))
+#   #  live_server_setup(live_server) # Setup on conftest per function
+    run_filter_test(client, live_server, '//*[@id="nope-doesnt-exist"]')
 
 # Test that notification is never sent
-
-def test_basic_markup_from_text(client, live_server, measure_memory_usage):
-    # Test the notification error templates convert to HTML if needed (link activate)
-    from ..notification.handler import markup_text_links_to_html
-    x = markup_text_links_to_html("hello https://google.com")
-    assert 'a href' in x

changedetectionio/tests/test_group.py

@@ -195,7 +195,7 @@ def test_group_tag_notification(client, live_server, measure_memory_usage):
                                                    "Diff as Patch: {{diff_patch}}\n"
                                                    ":-)",
                               "notification_screenshot": True,
-                              "notification_format": 'Plain Text',
+                              "notification_format": "Text",
                               "title": "test-tag"}
 
     res = client.post(

changedetectionio/tests/test_jinja2.py

@@ -1,10 +1,8 @@
 #!/usr/bin/env python3
 
 import time
-import arrow
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
-from ..jinja2_custom import render
 
 
 # def test_setup(client, live_server, measure_memory_usage):
@@ -35,35 +33,6 @@ def test_jinja2_in_url_query(client, live_server, measure_memory_usage):
     )
     assert b'date=2' in res.data
 
-# Test for issue #1493 - jinja2-time offset functionality
-def test_jinja2_time_offset_in_url_query(client, live_server, measure_memory_usage):
-    """Test that jinja2 time offset expressions work in watch URLs (issue #1493)."""
-
-    # Add our URL to the import page with time offset expression
-    test_url = url_for('test_return_query', _external=True)
-
-    # Test the exact syntax from issue #1493 that was broken in jinja2-time
-    # This should work now with our custom TimeExtension
-    full_url = "{}?{}".format(test_url,
-                              "timestamp={% now 'utc' - 'minutes=11', '%Y-%m-%d %H:%M' %}", )
-    res = client.post(
-        url_for("ui.ui_views.form_quick_watch_add"),
-        data={"url": full_url, "tags": "test"},
-        follow_redirects=True
-    )
-    assert b"Watch added" in res.data
-    wait_for_all_checks(client)
-
-    # Verify the URL was processed correctly (should not have errors)
-    res = client.get(
-        url_for("ui.ui_views.preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    # Should have a valid timestamp in the response
-    assert b'timestamp=' in res.data
-    # Should not have template error
-    assert b'Invalid template' not in res.data
-
 # https://techtonics.medium.com/secure-templating-with-jinja2-understanding-ssti-and-jinja2-sandbox-environment-b956edd60456
 def test_jinja2_security_url_query(client, live_server, measure_memory_usage):
     
@@ -87,243 +56,3 @@ def test_jinja2_security_url_query(client, live_server, measure_memory_usage):
     assert b'is invalid and cannot be used' in res.data
     # Some of the spewed output from the subclasses
     assert b'dict_values' not in res.data
-
-def test_timezone(mocker):
-    """Verify that timezone is parsed."""
-
-    timezone = 'America/Buenos_Aires'
-    currentDate = arrow.now(timezone)
-    arrowNowMock = mocker.patch("changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now")
-    arrowNowMock.return_value = currentDate
-    finalRender = render(f"{{% now '{timezone}' %}}")
-
-    assert finalRender == currentDate.strftime('%a, %d %b %Y %H:%M:%S')
-
-def test_format(mocker):
-    """Verify that format is parsed."""
-
-    timezone = 'utc'
-    format = '%d %b %Y %H:%M:%S'
-    currentDate = arrow.now(timezone)
-    arrowNowMock = mocker.patch("arrow.now")
-    arrowNowMock.return_value = currentDate
-    finalRender = render(f"{{% now '{timezone}', '{format}' %}}")
-
-    assert finalRender == currentDate.strftime(format)
-
-def test_add_time(environment):
-    """Verify that added time offset can be parsed."""
-
-    finalRender = render("{% now 'utc' + 'hours=2,seconds=30' %}")
-
-    assert finalRender == "Thu, 10 Dec 2015 01:33:31"
-
-def test_add_weekday(mocker):
-    """Verify that added weekday offset can be parsed."""
-
-    timezone = 'utc'
-    currentDate = arrow.now(timezone)
-    arrowNowMock = mocker.patch("changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now")
-    arrowNowMock.return_value = currentDate
-    finalRender = render(f"{{% now '{timezone}' + 'weekday=1' %}}")
-
-    assert finalRender == currentDate.shift(weekday=1).strftime('%a, %d %b %Y %H:%M:%S')
-
-
-def test_substract_time(environment):
-    """Verify that substracted time offset can be parsed."""
-
-    finalRender = render("{% now 'utc' - 'minutes=11' %}")
-
-    assert finalRender == "Wed, 09 Dec 2015 23:22:01"
-
-
-def test_offset_with_format(environment):
-    """Verify that offset works together with datetime format."""
-
-    finalRender = render(
-        "{% now 'utc' - 'days=2,minutes=33,seconds=1', '%d %b %Y %H:%M:%S' %}"
-    )
-
-    assert finalRender == "07 Dec 2015 23:00:00"
-
-def test_default_timezone_empty_string(environment):
-    """Verify that empty timezone string uses the default timezone (UTC in test environment)."""
-
-    # Empty string should use the default timezone which is 'UTC' (or from application settings)
-    finalRender = render("{% now '' %}")
-
-    # Should render with default format and UTC timezone (matches environment fixture)
-    assert finalRender == "Wed, 09 Dec 2015 23:33:01"
-
-def test_default_timezone_with_offset(environment):
-    """Verify that empty timezone works with offset operations."""
-
-    # Empty string with offset should use default timezone
-    finalRender = render("{% now '' + 'hours=2', '%d %b %Y %H:%M:%S' %}")
-
-    assert finalRender == "10 Dec 2015 01:33:01"
-
-def test_default_timezone_subtraction(environment):
-    """Verify that empty timezone works with subtraction offset."""
-
-    finalRender = render("{% now '' - 'minutes=11' %}")
-
-    assert finalRender == "Wed, 09 Dec 2015 23:22:01"
-
-def test_regex_replace_basic():
-    """Test basic regex_replace functionality."""
-
-    # Simple word replacement
-    finalRender = render("{{ 'hello world' | regex_replace('world', 'universe') }}")
-    assert finalRender == "hello universe"
-
-def test_regex_replace_with_groups():
-    """Test regex_replace with capture groups (issue #3501 use case)."""
-
-    # Transform HTML table data as described in the issue
-    template = "{{ '<td>thing</td><td>other</td>' | regex_replace('<td>([^<]+)</td><td>([^<]+)</td>', 'ThingLabel: \\\\1\\nOtherLabel: \\\\2') }}"
-    finalRender = render(template)
-    assert "ThingLabel: thing" in finalRender
-    assert "OtherLabel: other" in finalRender
-
-def test_regex_replace_multiple_matches():
-    """Test regex_replace replacing multiple occurrences."""
-
-    finalRender = render("{{ 'foo bar foo baz' | regex_replace('foo', 'qux') }}")
-    assert finalRender == "qux bar qux baz"
-
-def test_regex_replace_count_parameter():
-    """Test regex_replace with count parameter to limit replacements."""
-
-    finalRender = render("{{ 'foo bar foo baz' | regex_replace('foo', 'qux', 1) }}")
-    assert finalRender == "qux bar foo baz"
-
-def test_regex_replace_empty_replacement():
-    """Test regex_replace with empty replacement (removal)."""
-
-    finalRender = render("{{ 'hello world 123' | regex_replace('[0-9]+', '') }}")
-    assert finalRender == "hello world "
-
-def test_regex_replace_no_match():
-    """Test regex_replace when pattern doesn't match."""
-
-    finalRender = render("{{ 'hello world' | regex_replace('xyz', 'abc') }}")
-    assert finalRender == "hello world"
-
-def test_regex_replace_invalid_regex():
-    """Test regex_replace with invalid regex pattern returns original value."""
-
-    # Invalid regex (unmatched parenthesis)
-    finalRender = render("{{ 'hello world' | regex_replace('(invalid', 'replacement') }}")
-    assert finalRender == "hello world"
-
-def test_regex_replace_special_characters():
-    """Test regex_replace with special regex characters."""
-
-    finalRender = render("{{ 'Price: $50.00' | regex_replace('\\\\$([0-9.]+)', 'USD \\\\1') }}")
-    assert finalRender == "Price: USD 50.00"
-
-def test_regex_replace_multiline():
-    """Test regex_replace on multiline text."""
-
-    template = "{{ 'line1\\nline2\\nline3' | regex_replace('^line', 'row') }}"
-    finalRender = render(template)
-    # By default re.sub doesn't use MULTILINE flag, so only first line matches with ^
-    assert finalRender == "row1\nline2\nline3"
-
-def test_regex_replace_with_notification_context():
-    """Test regex_replace with notification diff variable."""
-
-    # Simulate how it would be used in notifications with diff variable
-    from changedetectionio.notification_service import NotificationContextData
-
-    context = NotificationContextData()
-    context['diff'] = '<td>value1</td><td>value2</td>'
-
-    template = "{{ diff | regex_replace('<td>([^<]+)</td>', '\\\\1 ') }}"
-
-    from changedetectionio.jinja2_custom import create_jinja_env
-    from jinja2 import BaseLoader
-
-    jinja2_env = create_jinja_env(loader=BaseLoader)
-    jinja2_env.globals.update(context)
-    finalRender = jinja2_env.from_string(template).render()
-
-    assert "value1 value2 " in finalRender
-
-def test_regex_replace_security_large_input():
-    """Test regex_replace handles large input safely."""
-
-    # Create a large input string (over 10MB)
-    large_input = "x" * (1024 * 1024 * 10 + 1000)
-    template = "{{ large_input | regex_replace('x', 'y') }}"
-
-    from changedetectionio.jinja2_custom import create_jinja_env
-    from jinja2 import BaseLoader
-
-    jinja2_env = create_jinja_env(loader=BaseLoader)
-    jinja2_env.globals['large_input'] = large_input
-    finalRender = jinja2_env.from_string(template).render()
-
-    # Should be truncated to 10MB
-    assert len(finalRender) == 1024 * 1024 * 10
-
-def test_regex_replace_security_long_pattern():
-    """Test regex_replace rejects very long patterns."""
-
-    # Pattern longer than 500 chars should be rejected
-    long_pattern = "a" * 501
-    finalRender = render("{{ 'test' | regex_replace('" + long_pattern + "', 'replacement') }}")
-
-    # Should return original value when pattern is too long
-    assert finalRender == "test"
-
-def test_regex_replace_security_dangerous_pattern():
-    """Test regex_replace detects and rejects dangerous nested quantifiers."""
-
-    # Patterns that could cause catastrophic backtracking
-    dangerous_patterns = [
-        "(a+)+",
-        "(a*)+",
-        "(a+)*",
-        "(a*)*",
-    ]
-
-    for dangerous in dangerous_patterns:
-        # Create a template with the dangerous pattern
-        # Using single quotes to avoid escaping issues
-        from changedetectionio.jinja2_custom import create_jinja_env
-        from jinja2 import BaseLoader
-
-        jinja2_env = create_jinja_env(loader=BaseLoader)
-        jinja2_env.globals['pattern'] = dangerous
-        template = "{{ 'aaaaaaaaaa' | regex_replace(pattern, 'x') }}"
-        finalRender = jinja2_env.from_string(template).render()
-
-        # Should return original value when dangerous pattern is detected
-        assert finalRender == "aaaaaaaaaa"
-
-def test_regex_replace_security_timeout_protection():
-    """Test that regex_replace has timeout protection (if SIGALRM available)."""
-    import signal
-
-    # Only test on systems that support SIGALRM
-    if not hasattr(signal, 'SIGALRM'):
-        # Skip test on Windows and other systems without SIGALRM
-        return
-
-    # This pattern is known to cause exponential backtracking on certain inputs
-    # but should be caught by our dangerous pattern detector
-    # We're mainly testing that the timeout mechanism works
-
-    from changedetectionio.jinja2_custom import regex_replace
-
-    # Create input that could trigger slow regex
-    test_input = "a" * 50 + "b"
-
-    # This shouldn't take long due to our protections
-    result = regex_replace(test_input, "a+b", "x")
-
-    # Should complete and return a result
-    assert result is not None

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -2,8 +2,7 @@
 # coding=utf-8
 
 import time
-from flask import url_for
-from markupsafe import escape
+from flask import url_for, escape
 from . util import live_server_setup, wait_for_all_checks, delete_all_watches
 import pytest
 jq_support = True

changedetectionio/tests/test_notification.py

@@ -101,7 +101,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
                                                    "Diff as Patch: {{diff_patch}}\n"
                                                    ":-)",
                               "notification_screenshot": True,
-                              "notification_format": 'Plain Text'}
+                              "notification_format": "Text"}
 
     notification_form_data.update({
         "url": test_url,
@@ -267,7 +267,7 @@ def test_notification_validation(client, live_server, measure_memory_usage):
 #        data={"notification_urls": 'json://localhost/foobar',
 #              "notification_title": "",
 #              "notification_body": "",
-#              "notification_format": 'Plain Text',
+#              "notification_format": "Text",
 #              "url": test_url,
 #              "tag": "my tag",
 #              "title": "my title",
@@ -284,27 +284,6 @@ def test_notification_validation(client, live_server, measure_memory_usage):
     )
 
 
-def test_notification_urls_jinja2_apprise_integration(client, live_server, measure_memory_usage):
-
-    #
-    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = "hassio://127.0.0.1/longaccesstoken?verify=no&nid={{watch_uuid}}"
-
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={
-              "application-fetch_backend": "html_requests",
-              "application-minutes_between_check": 180,
-              "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444, "somebug": "网站监测 内容更新了" }',
-              "application-notification_format": default_notification_format,
-              "application-notification_urls": test_notification_url,
-              # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-              "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }} ",
-              },
-        follow_redirects=True
-    )
-    assert b'Settings updated' in res.data
-
 
 def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_memory_usage):
     
@@ -315,7 +294,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&watch_uuid={{ watch_uuid }}&xxx={{ watch_url }}&now={% now 'Europe/London', '%Y-%m-%d' %}&+custom-header=123&+second=hello+world%20%22space%22"
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
 
     res = client.post(
         url_for("settings.settings_page"),
@@ -341,7 +320,6 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     )
 
     assert b"Watch added" in res.data
-    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     wait_for_all_checks(client)
     set_modified_response()
@@ -371,11 +349,6 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
         assert 'xxx=http' in notification_url
         # apprise style headers should be stripped
         assert 'custom-header' not in notification_url
-        # Check jinja2 custom arrow/jinja2-time replace worked
-        assert 'now=2' in notification_url
-        # Check our watch_uuid appeared
-        assert f'watch_uuid={watch_uuid}' in notification_url
-
 
     with open("test-datastore/notification-headers.txt", 'r') as f:
         notification_headers = f.read()
@@ -383,7 +356,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
         assert 'second: hello world "space"' in notification_headers.lower()
 
 
-    # Should always be automatically detected as JSON content type even when we set it as 'Plain Text' (default)
+    # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
     assert os.path.isfile("test-datastore/notification-content-type.txt")
     with open("test-datastore/notification-content-type.txt", 'r') as f:
         assert 'application/json' in f.read()
@@ -443,6 +416,7 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert res.status_code != 400
     assert res.status_code != 500
 
+
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
         assert test_body in x
@@ -485,7 +459,7 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
 
 def _test_color_notifications(client, notification_body_token):
 
-    from changedetectionio.diff import HTML_ADDED_STYLE, HTML_REMOVED_STYLE
+    from changedetectionio.diff import ADDED_STYLE, REMOVED_STYLE
 
     set_original_response()
 
@@ -533,7 +507,7 @@ def _test_color_notifications(client, notification_body_token):
 
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
-        assert f'<span style="{HTML_REMOVED_STYLE}">Which is across multiple lines' in x
+        assert f'<span style="{REMOVED_STYLE}">Which is across multiple lines' in x
 
 
     client.get(
@@ -541,7 +515,9 @@ def _test_color_notifications(client, notification_body_token):
         follow_redirects=True
     )
 
-# Just checks the format of the colour notifications was correct
 def test_html_color_notifications(client, live_server, measure_memory_usage):
+
+    
     _test_color_notifications(client, '{{diff}}')
     _test_color_notifications(client, '{{diff_full}}')
+    

changedetectionio/tests/test_notification_errors.py

@@ -30,7 +30,7 @@ def test_check_notification_error_handling(client, live_server, measure_memory_u
         data={"notification_urls": f"{broken_notification_url}\r\n{working_notification_url}",
               "notification_title": "xxx",
               "notification_body": "xxxxx",
-              "notification_format": 'Plain Text',
+              "notification_format": "Text",
               "url": test_url,
               "tags": "",
               "title": "",

changedetectionio/tests/test_rss.py

@@ -110,9 +110,8 @@ def test_basic_cdata_rss_markup(client, live_server, measure_memory_usage):
     
 
     set_original_cdata_xml()
-    # Rarely do endpoints give the right header, usually just text/xml, so we check also for <rss
-    # This also triggers the automatic CDATA text parser so the RSS goes back a nice content list
-    test_url = url_for('test_endpoint', content_type="text/xml; charset=UTF-8", _external=True)
+
+    test_url = url_for('test_endpoint', content_type="application/atom+xml; charset=UTF-8", _external=True)
 
     # Add our URL to the import page
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)

changedetectionio/tests/test_rss_reader_mode.py

@@ -1,98 +0,0 @@
-#!/usr/bin/env python3
-
-import time
-from flask import url_for
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
-    extract_UUID_from_client, delete_all_watches
-
-
-def set_original_cdata_xml():
-    test_return_data = """<rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
-<channel>
-<title>Security Bulletins on wetscale</title>
-<link>https://wetscale.com/security-bulletins/</link>
-<description>Recent security bulletins from wetscale</description>
-<lastBuildDate>Fri, 10 Oct 2025 14:58:11 GMT</lastBuildDate>
-<docs>https://validator.w3.org/feed/docs/rss2.html</docs>
-<generator>wetscale.com</generator>
-<language>en-US</language>
-<copyright>© 2025 wetscale Inc. All rights reserved.</copyright>
-<atom:link href="https://wetscale.com/security-bulletins/index.xml" rel="self" type="application/rss+xml"/>
-<item>
-<title>TS-2025-005</title>
-<link>https://wetscale.com/security-bulletins/#ts-2025-005</link>
-<guid>https://wetscale.com/security-bulletins/#ts-2025-005</guid>
-<pubDate>Thu, 07 Aug 2025 00:00:00 GMT</pubDate>
-<description><p>Wet noodles escape<br><p>they also found themselves outside</p> </description>
-</item>
-
-
-<item>
-<title>TS-2025-004</title>
-<link>https://wetscale.com/security-bulletins/#ts-2025-004</link>
-<guid>https://wetscale.com/security-bulletins/#ts-2025-004</guid>
-<pubDate>Tue, 27 May 2025 00:00:00 GMT</pubDate>
-<description>
-    <![CDATA[ <img class="type:primaryImage" src="https://testsite.com/701c981da04869e.jpg"/><p>The days of Terminator and The Matrix could be closer. But be positive.</p><p><a href="https://testsite.com">Read more link...</a></p> ]]>
-</description>
-</item>
-    </channel>
-    </rss>
-            """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-
-
-def test_rss_reader_mode(client, live_server, measure_memory_usage):
-    set_original_cdata_xml()
-
-    # Rarely do endpoints give the right header, usually just text/xml, so we check also for <rss
-    # This also triggers the automatic CDATA text parser so the RSS goes back a nice content list
-    test_url = url_for('test_endpoint', content_type="text/xml; charset=UTF-8", _external=True)
-    live_server.app.config['DATASTORE'].data['settings']['application']['rss_reader_mode'] = True
-
-
-    # Add our URL to the import page
-    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-
-    wait_for_all_checks(client)
-
-
-    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
-    dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(dates[0])
-    assert 'Wet noodles escape' in snapshot_contents
-    assert '<br>' not in snapshot_contents
-    assert '&lt;' not in snapshot_contents
-    assert 'The days of Terminator and The Matrix' in snapshot_contents
-    assert 'PubDate: Thu, 07 Aug 2025 00:00:00 GMT' in snapshot_contents
-    delete_all_watches(client)
-
-def test_rss_reader_mode_with_css_filters(client, live_server, measure_memory_usage):
-    set_original_cdata_xml()
-
-    # Rarely do endpoints give the right header, usually just text/xml, so we check also for <rss
-    # This also triggers the automatic CDATA text parser so the RSS goes back a nice content list
-    test_url = url_for('test_endpoint', content_type="text/xml; charset=UTF-8", _external=True)
-    live_server.app.config['DATASTORE'].data['settings']['application']['rss_reader_mode'] = True
-
-
-    # Add our URL to the import page
-    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url, extras={'include_filters': [".last"]})
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-
-    wait_for_all_checks(client)
-
-
-    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
-    dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(dates[0])
-    assert 'Wet noodles escape' not in snapshot_contents
-    assert '<br>' not in snapshot_contents
-    assert '&lt;' not in snapshot_contents
-    assert 'The days of Terminator and The Matrix' in snapshot_contents
-    delete_all_watches(client)
-

changedetectionio/tests/test_scheduler.py

@@ -24,7 +24,7 @@ def test_check_basic_scheduler_functionality(client, live_server, measure_memory
         url_for("settings.settings_page"),
         data={"application-empty_pages_are_a_change": "",
               "requests-time_between_check-seconds": 1,
-              "application-scheduler_timezone_default": "Pacific/Kiritimati",  # Most Forward Time Zone (UTC+14:00)
+              "application-timezone": "Pacific/Kiritimati",  # Most Forward Time Zone (UTC+14:00)
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
     )
@@ -119,7 +119,7 @@ def test_check_basic_global_scheduler_functionality(client, live_server, measure
 
     data = {
         "application-empty_pages_are_a_change": "",
-        "application-scheduler_timezone_default": "Pacific/Kiritimati",  # Most Forward Time Zone (UTC+14:00)
+        "application-timezone": "Pacific/Kiritimati",  # Most Forward Time Zone (UTC+14:00)
         'application-fetch_backend': "html_requests",
         "requests-time_between_check-hours": 0,
         "requests-time_between_check-minutes": 0,

changedetectionio/tests/test_xpath_default_namespace.py

@@ -1,153 +0,0 @@
-#!/usr/bin/env python3
-"""
-Unit tests for XPath default namespace handling in RSS/Atom feeds.
-Tests the fix for issue where //title/text() returns empty on feeds with default namespaces.
-
-Real-world test data from https://github.com/microsoft/PowerToys/releases.atom
-"""
-
-import sys
-import os
-import pytest
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-import html_tools
-
-
-# Real-world Atom feed with default namespace from GitHub PowerToys releases
-# This is the actual format that was failing before the fix
-atom_feed_with_default_ns = """<?xml version="1.0" encoding="UTF-8"?>
-<feed xmlns="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/" xml:lang="en-US">
-  <id>tag:github.com,2008:https://github.com/microsoft/PowerToys/releases</id>
-  <link type="text/html" rel="alternate" href="https://github.com/microsoft/PowerToys/releases"/>
-  <link type="application/atom+xml" rel="self" href="https://github.com/microsoft/PowerToys/releases.atom"/>
-  <title>Release notes from PowerToys</title>
-  <updated>2025-10-23T08:53:12Z</updated>
-  <entry>
-    <id>tag:github.com,2008:Repository/184456251/v0.95.1</id>
-    <updated>2025-10-24T14:20:14Z</updated>
-    <link rel="alternate" type="text/html" href="https://github.com/microsoft/PowerToys/releases/tag/v0.95.1"/>
-    <title>Release 0.95.1</title>
-    <content type="html">&lt;p&gt;This patch release fixes several important stability issues.&lt;/p&gt;</content>
-    <author>
-      <name>Jaylyn-Barbee</name>
-    </author>
-  </entry>
-  <entry>
-    <id>tag:github.com,2008:Repository/184456251/v0.95.0</id>
-    <updated>2025-10-17T12:51:21Z</updated>
-    <link rel="alternate" type="text/html" href="https://github.com/microsoft/PowerToys/releases/tag/v0.95.0"/>
-    <title>Release v0.95.0</title>
-    <content type="html">&lt;p&gt;New features, stability, optimization improvements.&lt;/p&gt;</content>
-    <author>
-      <name>Jaylyn-Barbee</name>
-    </author>
-  </entry>
-</feed>"""
-
-# RSS feed without default namespace
-rss_feed_no_default_ns = """<?xml version="1.0" encoding="UTF-8"?>
-<rss version="2.0">
-  <channel>
-    <title>Channel Title</title>
-    <description>Channel Description</description>
-    <item>
-      <title>Item 1 Title</title>
-      <description>Item 1 Description</description>
-    </item>
-    <item>
-      <title>Item 2 Title</title>
-      <description>Item 2 Description</description>
-    </item>
-  </channel>
-</rss>"""
-
-# RSS 2.0 feed with namespace prefix (not default)
-rss_feed_with_ns_prefix = """<?xml version="1.0" encoding="UTF-8"?>
-<rss xmlns:dc="http://purl.org/dc/elements/1.1/"
-     xmlns:content="http://purl.org/rss/1.0/modules/content/"
-     xmlns:atom="http://www.w3.org/2005/Atom"
-     version="2.0">
-  <channel>
-    <title>Channel Title</title>
-    <atom:link href="http://example.com/feed" rel="self" type="application/rss+xml"/>
-    <item>
-      <title>Item Title</title>
-      <dc:creator>Author Name</dc:creator>
-    </item>
-  </channel>
-</rss>"""
-
-
-class TestXPathDefaultNamespace:
-    """Test XPath queries on feeds with and without default namespaces."""
-
-    def test_atom_feed_simple_xpath_with_xpath_filter(self):
-        """Test that //title/text() works on Atom feed with default namespace using xpath_filter."""
-        result = html_tools.xpath_filter('//title/text()', atom_feed_with_default_ns, is_rss=True)
-        assert 'Release notes from PowerToys' in result
-        assert 'Release 0.95.1' in result
-        assert 'Release v0.95.0' in result
-
-    def test_atom_feed_nested_xpath_with_xpath_filter(self):
-        """Test nested XPath like //entry/title/text() on Atom feed."""
-        result = html_tools.xpath_filter('//entry/title/text()', atom_feed_with_default_ns, is_rss=True)
-        assert 'Release 0.95.1' in result
-        assert 'Release v0.95.0' in result
-        # Should NOT include the feed title
-        assert 'Release notes from PowerToys' not in result
-
-    def test_atom_feed_other_elements_with_xpath_filter(self):
-        """Test that other elements like //updated/text() work on Atom feed."""
-        result = html_tools.xpath_filter('//updated/text()', atom_feed_with_default_ns, is_rss=True)
-        assert '2025-10-23T08:53:12Z' in result
-        assert '2025-10-24T14:20:14Z' in result
-
-    def test_rss_feed_without_namespace(self):
-        """Test that //title/text() works on RSS feed without default namespace."""
-        result = html_tools.xpath_filter('//title/text()', rss_feed_no_default_ns, is_rss=True)
-        assert 'Channel Title' in result
-        assert 'Item 1 Title' in result
-        assert 'Item 2 Title' in result
-
-    def test_rss_feed_nested_xpath(self):
-        """Test nested XPath on RSS feed without default namespace."""
-        result = html_tools.xpath_filter('//item/title/text()', rss_feed_no_default_ns, is_rss=True)
-        assert 'Item 1 Title' in result
-        assert 'Item 2 Title' in result
-        # Should NOT include channel title
-        assert 'Channel Title' not in result
-
-    def test_rss_feed_with_prefixed_namespaces(self):
-        """Test that feeds with namespace prefixes (not default) still work."""
-        result = html_tools.xpath_filter('//title/text()', rss_feed_with_ns_prefix, is_rss=True)
-        assert 'Channel Title' in result
-        assert 'Item Title' in result
-
-    def test_local_name_workaround_still_works(self):
-        """Test that local-name() workaround still works for Atom feeds."""
-        result = html_tools.xpath_filter('//*[local-name()="title"]/text()', atom_feed_with_default_ns, is_rss=True)
-        assert 'Release notes from PowerToys' in result
-        assert 'Release 0.95.1' in result
-
-    def test_xpath1_filter_without_default_namespace(self):
-        """Test xpath1_filter works on RSS without default namespace."""
-        result = html_tools.xpath1_filter('//title/text()', rss_feed_no_default_ns, is_rss=True)
-        assert 'Channel Title' in result
-        assert 'Item 1 Title' in result
-
-    def test_xpath1_filter_with_default_namespace_returns_empty(self):
-        """Test that xpath1_filter returns empty on Atom with default namespace (known limitation)."""
-        result = html_tools.xpath1_filter('//title/text()', atom_feed_with_default_ns, is_rss=True)
-        # xpath1_filter (lxml) doesn't support default namespaces, so this returns empty
-        assert result == ''
-
-    def test_xpath1_filter_local_name_workaround(self):
-        """Test that xpath1_filter works with local-name() workaround on Atom feeds."""
-        result = html_tools.xpath1_filter('//*[local-name()="title"]/text()', atom_feed_with_default_ns, is_rss=True)
-        assert 'Release notes from PowerToys' in result
-        assert 'Release 0.95.1' in result
-
-
-if __name__ == '__main__':
-    pytest.main([__file__, '-v'])

changedetectionio/tests/unit/test_jinja2_security.py

@@ -4,7 +4,7 @@
 # python3 -m unittest changedetectionio.tests.unit.test_jinja2_security
 
 import unittest
-from changedetectionio import jinja2_custom as safe_jinja
+from changedetectionio import safe_jinja
 
 
 # mostly

changedetectionio/tests/unit/test_notification_diff.py

@@ -24,18 +24,18 @@ class TestDiffBuilder(unittest.TestCase):
 
         output = output.split("\n")
 
-        # Check that placeholders are present (they get replaced in apply_service_tweaks)
-        self.assertTrue(any(diff.CHANGED_PLACEMARKER_OPEN in line and 'ok' in line for line in output))
-        self.assertTrue(any(diff.CHANGED_INTO_PLACEMARKER_OPEN in line and 'xok' in line for line in output))
-        self.assertTrue(any(diff.CHANGED_INTO_PLACEMARKER_OPEN in line and 'next-x-ok' in line for line in output))
-        self.assertTrue(any(diff.ADDED_PLACEMARKER_OPEN in line and 'and something new' in line for line in output))
+
+        self.assertIn('(changed) ok', output)
+        self.assertIn('(into) xok', output)
+        self.assertIn('(into) next-x-ok', output)
+        self.assertIn('(added) and something new', output)
 
         with open(base_dir + "/test-content/after-2.txt", 'r') as f:
             newest_version_file_contents = f.read()
         output = diff.render_diff(previous_version_file_contents, newest_version_file_contents)
         output = output.split("\n")
-        self.assertTrue(any(diff.REMOVED_PLACEMARKER_OPEN in line and 'for having learned computerese,' in line for line in output))
-        self.assertTrue(any(diff.REMOVED_PLACEMARKER_OPEN in line and 'I continue to examine bits, bytes and words' in line for line in output))
+        self.assertIn('(removed) for having learned computerese,', output)
+        self.assertIn('(removed) I continue to examine bits, bytes and words', output)
 
         #diff_removed
         with open(base_dir + "/test-content/before.txt", 'r') as f:
@@ -45,18 +45,18 @@ class TestDiffBuilder(unittest.TestCase):
             newest_version_file_contents = f.read()
         output = diff.render_diff(previous_version_file_contents, newest_version_file_contents, include_equal=False, include_removed=True, include_added=False)
         output = output.split("\n")
-        self.assertTrue(any(diff.CHANGED_PLACEMARKER_OPEN in line and 'ok' in line for line in output))
-        self.assertTrue(any(diff.CHANGED_INTO_PLACEMARKER_OPEN in line and 'xok' in line for line in output))
-        self.assertTrue(any(diff.CHANGED_INTO_PLACEMARKER_OPEN in line and 'next-x-ok' in line for line in output))
-        self.assertFalse(any(diff.ADDED_PLACEMARKER_OPEN in line and 'and something new' in line for line in output))
+        self.assertIn('(changed) ok', output)
+        self.assertIn('(into) xok', output)
+        self.assertIn('(into) next-x-ok', output)
+        self.assertNotIn('(added) and something new', output)
 
         #diff_removed
         with open(base_dir + "/test-content/after-2.txt", 'r') as f:
             newest_version_file_contents = f.read()
         output = diff.render_diff(previous_version_file_contents, newest_version_file_contents, include_equal=False, include_removed=True, include_added=False)
         output = output.split("\n")
-        self.assertTrue(any(diff.REMOVED_PLACEMARKER_OPEN in line and 'for having learned computerese,' in line for line in output))
-        self.assertTrue(any(diff.REMOVED_PLACEMARKER_OPEN in line and 'I continue to examine bits, bytes and words' in line for line in output))
+        self.assertIn('(removed) for having learned computerese,', output)
+        self.assertIn('(removed) I continue to examine bits, bytes and words', output)
 
     def test_expected_diff_patch_output(self):
         base_dir = os.path.dirname(__file__)

changedetectionio/tests/unit/test_restock_logic.py

@@ -6,7 +6,7 @@
 import unittest
 import os
 
-import changedetectionio.processors.restock_diff.processor as restock_diff
+from changedetectionio.processors import restock_diff
 
 # mostly
 class TestDiffBuilder(unittest.TestCase):

changedetectionio/tests/unit/test_scheduler.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 
 # run from dir above changedetectionio/ dir
-# python3 -m unittest changedetectionio.tests.unit.test_scheduler
+# python3 -m unittest changedetectionio.tests.unit.test_jinja2_security
 
 import unittest
-import arrow
+from datetime import datetime, timedelta
+from zoneinfo import ZoneInfo
 
 class TestScheduler(unittest.TestCase):
 
@@ -12,13 +13,12 @@ class TestScheduler(unittest.TestCase):
     # UTC-12:00 (Baker Island, Howland Island) is the farthest behind, always one calendar day behind UTC.
 
     def test_timezone_basic_time_within_schedule(self):
-        """Test that current time is detected as within schedule window."""
         from changedetectionio import time_handler
 
         timezone_str = 'Europe/Berlin'
-        debug_datetime = arrow.now(timezone_str)
-        day_of_week = debug_datetime.format('dddd')
-        time_str = debug_datetime.format('HH:00')
+        debug_datetime = datetime.now(ZoneInfo(timezone_str))
+        day_of_week = debug_datetime.strftime('%A')
+        time_str = str(debug_datetime.hour)+':00'
         duration = 60  # minutes
 
         # The current time should always be within 60 minutes of [time_hour]:00
@@ -30,17 +30,16 @@ class TestScheduler(unittest.TestCase):
         self.assertEqual(result, True, f"{debug_datetime} is within time scheduler {day_of_week} {time_str} in {timezone_str} for {duration} minutes")
 
     def test_timezone_basic_time_outside_schedule(self):
-        """Test that time from yesterday is outside current schedule."""
         from changedetectionio import time_handler
 
         timezone_str = 'Europe/Berlin'
-        # We try a date in the past (yesterday)
-        debug_datetime = arrow.now(timezone_str).shift(days=-1)
-        day_of_week = debug_datetime.format('dddd')
-        time_str = debug_datetime.format('HH:00')
-        duration = 60 * 24  # minutes
+        # We try a date in the future..
+        debug_datetime = datetime.now(ZoneInfo(timezone_str))+ timedelta(days=-1)
+        day_of_week = debug_datetime.strftime('%A')
+        time_str = str(debug_datetime.hour) + ':00'
+        duration = 60*24  # minutes
 
-        # The current time should NOT be within yesterday's schedule
+        # The current time should always be within 60 minutes of [time_hour]:00
         result = time_handler.am_i_inside_time(day_of_week=day_of_week,
                                                time_str=time_str,
                                                timezone_str=timezone_str,
@@ -49,58 +48,6 @@ class TestScheduler(unittest.TestCase):
         self.assertNotEqual(result, True,
                          f"{debug_datetime} is NOT within time scheduler {day_of_week} {time_str} in {timezone_str} for {duration} minutes")
 
-    def test_timezone_utc_within_schedule(self):
-        """Test UTC timezone works correctly."""
-        from changedetectionio import time_handler
-
-        timezone_str = 'UTC'
-        debug_datetime = arrow.now(timezone_str)
-        day_of_week = debug_datetime.format('dddd')
-        time_str = debug_datetime.format('HH:00')
-        duration = 120  # minutes
-
-        result = time_handler.am_i_inside_time(day_of_week=day_of_week,
-                                               time_str=time_str,
-                                               timezone_str=timezone_str,
-                                               duration=duration)
-
-        self.assertTrue(result, "Current time should be within UTC schedule")
-
-    def test_timezone_extreme_ahead(self):
-        """Test with UTC+14 timezone (Line Islands, Kiribati)."""
-        from changedetectionio import time_handler
-
-        timezone_str = 'Pacific/Kiritimati'  # UTC+14
-        debug_datetime = arrow.now(timezone_str)
-        day_of_week = debug_datetime.format('dddd')
-        time_str = debug_datetime.format('HH:00')
-        duration = 60
-
-        result = time_handler.am_i_inside_time(day_of_week=day_of_week,
-                                               time_str=time_str,
-                                               timezone_str=timezone_str,
-                                               duration=duration)
-
-        self.assertTrue(result, "Should work with extreme ahead timezone")
-
-    def test_timezone_extreme_behind(self):
-        """Test with UTC-12 timezone (Baker Island)."""
-        from changedetectionio import time_handler
-
-        # Using Etc/GMT+12 which is UTC-12 (confusing, but that's how it works)
-        timezone_str = 'Etc/GMT+12'  # UTC-12
-        debug_datetime = arrow.now(timezone_str)
-        day_of_week = debug_datetime.format('dddd')
-        time_str = debug_datetime.format('HH:00')
-        duration = 60
-
-        result = time_handler.am_i_inside_time(day_of_week=day_of_week,
-                                               time_str=time_str,
-                                               timezone_str=timezone_str,
-                                               duration=duration)
-
-        self.assertTrue(result, "Should work with extreme behind timezone")
-
 
 if __name__ == '__main__':
     unittest.main()

changedetectionio/tests/unit/test_time_extension.py

@@ -1,138 +0,0 @@
-#!/usr/bin/env python3
-"""
-Simple unit tests for TimeExtension that mimic how safe_jinja.py uses it.
-These tests demonstrate that the environment.default_timezone override works
-exactly as intended in the actual application code.
-"""
-
-import arrow
-from jinja2.sandbox import ImmutableSandboxedEnvironment
-from changedetectionio.jinja2_custom.extensions.TimeExtension import TimeExtension
-
-
-def test_default_timezone_override_like_safe_jinja(mocker):
-    """
-    Test that mirrors exactly how safe_jinja.py uses the TimeExtension.
-    This is the simplest demonstration that environment.default_timezone works.
-    """
-    # Create environment (TimeExtension.__init__ sets default_timezone='UTC')
-    jinja2_env = ImmutableSandboxedEnvironment(extensions=[TimeExtension])
-
-    # Override the default timezone - exactly like safe_jinja.py does
-    jinja2_env.default_timezone = 'America/New_York'
-
-    # Mock arrow.now to return a fixed time
-    fixed_time = arrow.Arrow(2025, 1, 15, 12, 0, 0, tzinfo='America/New_York')
-    mock = mocker.patch("changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now", return_value=fixed_time)
-
-    # Use empty string timezone - should use the overridden default
-    template_str = "{% now '' %}"
-    output = jinja2_env.from_string(template_str).render()
-
-    # Verify arrow.now was called with the overridden timezone
-    mock.assert_called_with('America/New_York')
-    assert '2025' in output
-    assert 'Jan' in output
-
-
-def test_default_timezone_not_overridden(mocker):
-    """
-    Test that without override, the default 'UTC' from __init__ is used.
-    """
-    # Create environment (TimeExtension.__init__ sets default_timezone='UTC')
-    jinja2_env = ImmutableSandboxedEnvironment(extensions=[TimeExtension])
-
-    # DON'T override - should use 'UTC' default
-
-    # Mock arrow.now
-    fixed_time = arrow.Arrow(2025, 1, 15, 17, 0, 0, tzinfo='UTC')
-    mock = mocker.patch("changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now", return_value=fixed_time)
-
-    # Use empty string timezone - should use 'UTC' default
-    template_str = "{% now '' %}"
-    output = jinja2_env.from_string(template_str).render()
-
-    # Verify arrow.now was called with 'UTC'
-    mock.assert_called_with('UTC')
-    assert '2025' in output
-
-
-def test_datetime_format_override_like_safe_jinja(mocker):
-    """
-    Test that environment.datetime_format can be overridden after creation.
-    """
-    # Create environment (default format is '%a, %d %b %Y %H:%M:%S')
-    jinja2_env = ImmutableSandboxedEnvironment(extensions=[TimeExtension])
-
-    # Override the datetime format
-    jinja2_env.datetime_format = '%Y-%m-%d %H:%M:%S'
-
-    # Mock arrow.now
-    fixed_time = arrow.Arrow(2025, 1, 15, 14, 30, 45, tzinfo='UTC')
-    mocker.patch("changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now", return_value=fixed_time)
-
-    # Don't specify format - should use overridden default
-    template_str = "{% now 'UTC' %}"
-    output = jinja2_env.from_string(template_str).render()
-
-    # Should use custom format YYYY-MM-DD HH:MM:SS
-    assert output == '2025-01-15 14:30:45'
-
-
-def test_offset_with_overridden_timezone(mocker):
-    """
-    Test that offset operations also respect the overridden default_timezone.
-    """
-    jinja2_env = ImmutableSandboxedEnvironment(extensions=[TimeExtension])
-
-    # Override to use Europe/London
-    jinja2_env.default_timezone = 'Europe/London'
-
-    fixed_time = arrow.Arrow(2025, 1, 15, 10, 0, 0, tzinfo='Europe/London')
-    mock = mocker.patch("changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now", return_value=fixed_time)
-
-    # Use offset with empty timezone string
-    template_str = "{% now '' + 'hours=2', '%Y-%m-%d %H:%M:%S' %}"
-    output = jinja2_env.from_string(template_str).render()
-
-    # Should have called arrow.now with Europe/London
-    mock.assert_called_with('Europe/London')
-    # Should be 10:00 + 2 hours = 12:00
-    assert output == '2025-01-15 12:00:00'
-
-
-def test_weekday_parameter_converted_to_int(mocker):
-    """
-    Test that weekday parameter is properly converted from float to int.
-    This is important because arrow.shift() requires weekday as int, not float.
-    """
-    jinja2_env = ImmutableSandboxedEnvironment(extensions=[TimeExtension])
-
-    # Wednesday, Jan 15, 2025
-    fixed_time = arrow.Arrow(2025, 1, 15, 12, 0, 0, tzinfo='UTC')
-    mocker.patch("changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now", return_value=fixed_time)
-
-    # Add offset to next Monday (weekday=0)
-    template_str = "{% now 'UTC' + 'weekday=0', '%A' %}"
-    output = jinja2_env.from_string(template_str).render()
-
-    # Should be Monday
-    assert output == 'Monday'
-
-
-def test_multiple_offset_parameters(mocker):
-    """
-    Test that multiple offset parameters can be combined in one expression.
-    """
-    jinja2_env = ImmutableSandboxedEnvironment(extensions=[TimeExtension])
-
-    fixed_time = arrow.Arrow(2025, 1, 15, 10, 30, 45, tzinfo='UTC')
-    mocker.patch("changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now", return_value=fixed_time)
-
-    # Test multiple parameters: days, hours, minutes, seconds
-    template_str = "{% now 'UTC' + 'days=1,hours=2,minutes=15,seconds=10', '%Y-%m-%d %H:%M:%S' %}"
-    output = jinja2_env.from_string(template_str).render()
-
-    # 2025-01-15 10:30:45 + 1 day + 2 hours + 15 minutes + 10 seconds
-    # = 2025-01-16 12:45:55
-    assert output == '2025-01-16 12:45:55'

changedetectionio/tests/unit/test_time_handler.py

@@ -1,429 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-Comprehensive tests for time_handler module refactored to use arrow.
-
-Run from project root:
-python3 -m pytest changedetectionio/tests/unit/test_time_handler.py -v
-"""
-
-import unittest
-import arrow
-from changedetectionio import time_handler
-
-
-class TestAmIInsideTime(unittest.TestCase):
-    """Tests for the am_i_inside_time function."""
-
-    def test_current_time_within_schedule(self):
-        """Test that current time is detected as within schedule."""
-        # Get current time in a specific timezone
-        timezone_str = 'Europe/Berlin'
-        now = arrow.now(timezone_str)
-        day_of_week = now.format('dddd')
-        time_str = now.format('HH:00')  # Current hour, 0 minutes
-        duration = 60  # 60 minutes
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        self.assertTrue(result, f"Current time should be within {duration} minute window starting at {time_str}")
-
-    def test_current_time_outside_schedule(self):
-        """Test that time in the past is not within current schedule."""
-        timezone_str = 'Europe/Berlin'
-        # Get yesterday's date
-        yesterday = arrow.now(timezone_str).shift(days=-1)
-        day_of_week = yesterday.format('dddd')
-        time_str = yesterday.format('HH:mm')
-        duration = 30  # Only 30 minutes
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        self.assertFalse(result, "Yesterday's time should not be within current schedule")
-
-    def test_timezone_pacific_within_schedule(self):
-        """Test with US/Pacific timezone."""
-        timezone_str = 'US/Pacific'
-        now = arrow.now(timezone_str)
-        day_of_week = now.format('dddd')
-        time_str = now.format('HH:00')
-        duration = 120  # 2 hours
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        self.assertTrue(result)
-
-    def test_timezone_tokyo_within_schedule(self):
-        """Test with Asia/Tokyo timezone."""
-        timezone_str = 'Asia/Tokyo'
-        now = arrow.now(timezone_str)
-        day_of_week = now.format('dddd')
-        time_str = now.format('HH:00')
-        duration = 90  # 1.5 hours
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        self.assertTrue(result)
-
-    def test_schedule_crossing_midnight(self):
-        """Test schedule that crosses midnight."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-
-        # Set schedule to start 23:30 with 120 minute duration (crosses midnight)
-        day_of_week = now.format('dddd')
-        time_str = "23:30"
-        duration = 120  # 2 hours - goes into next day
-
-        # If we're at 00:15 the next day, we should still be in the schedule
-        if now.hour == 0 and now.minute < 30:
-            # We're in the time window that spilled over from yesterday
-            result = time_handler.am_i_inside_time(
-                day_of_week=day_of_week,
-                time_str=time_str,
-                timezone_str=timezone_str,
-                duration=duration
-            )
-            # This might be true or false depending on exact time
-            self.assertIsInstance(result, bool)
-
-    def test_invalid_day_of_week(self):
-        """Test that invalid day raises ValueError."""
-        with self.assertRaises(ValueError) as context:
-            time_handler.am_i_inside_time(
-                day_of_week="Funday",
-                time_str="12:00",
-                timezone_str="UTC",
-                duration=60
-            )
-        self.assertIn("Invalid day_of_week", str(context.exception))
-
-    def test_invalid_time_format(self):
-        """Test that invalid time format raises ValueError."""
-        with self.assertRaises(ValueError) as context:
-            time_handler.am_i_inside_time(
-                day_of_week="Monday",
-                time_str="25:99",
-                timezone_str="UTC",
-                duration=60
-            )
-        self.assertIn("Invalid time_str", str(context.exception))
-
-    def test_invalid_time_format_non_numeric(self):
-        """Test that non-numeric time raises ValueError."""
-        with self.assertRaises(ValueError) as context:
-            time_handler.am_i_inside_time(
-                day_of_week="Monday",
-                time_str="twelve:thirty",
-                timezone_str="UTC",
-                duration=60
-            )
-        self.assertIn("Invalid time_str", str(context.exception))
-
-    def test_invalid_timezone(self):
-        """Test that invalid timezone raises ValueError."""
-        with self.assertRaises(ValueError) as context:
-            time_handler.am_i_inside_time(
-                day_of_week="Monday",
-                time_str="12:00",
-                timezone_str="Invalid/Timezone",
-                duration=60
-            )
-        self.assertIn("Invalid timezone_str", str(context.exception))
-
-    def test_short_duration(self):
-        """Test with very short duration (15 minutes default)."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        day_of_week = now.format('dddd')
-        time_str = now.format('HH:mm')
-        duration = 15  # Default duration
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        self.assertTrue(result, "Current time should be within 15 minute window")
-
-    def test_long_duration(self):
-        """Test with long duration (24 hours)."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        day_of_week = now.format('dddd')
-        # Set time to current hour
-        time_str = now.format('HH:00')
-        duration = 1440  # 24 hours in minutes
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        self.assertTrue(result, "Current time should be within 24 hour window")
-
-    def test_case_insensitive_day(self):
-        """Test that day of week is case insensitive."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        day_of_week = now.format('dddd').lower()  # lowercase day
-        time_str = now.format('HH:00')
-        duration = 60
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        self.assertTrue(result, "Lowercase day should work")
-
-    def test_edge_case_midnight(self):
-        """Test edge case at exactly midnight."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        day_of_week = now.format('dddd')
-        time_str = "00:00"
-        duration = 60
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        # Should be true if we're in the first hour of the day
-        if now.hour == 0:
-            self.assertTrue(result)
-
-    def test_edge_case_end_of_day(self):
-        """Test edge case near end of day."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        day_of_week = now.format('dddd')
-        time_str = "23:45"
-        duration = 30  # 30 minutes crosses midnight
-
-        result = time_handler.am_i_inside_time(
-            day_of_week=day_of_week,
-            time_str=time_str,
-            timezone_str=timezone_str,
-            duration=duration
-        )
-
-        # Result depends on current time
-        self.assertIsInstance(result, bool)
-
-
-class TestIsWithinSchedule(unittest.TestCase):
-    """Tests for the is_within_schedule function."""
-
-    def test_schedule_disabled(self):
-        """Test that disabled schedule returns False."""
-        time_schedule_limit = {'enabled': False}
-        result = time_handler.is_within_schedule(time_schedule_limit)
-        self.assertFalse(result)
-
-    def test_schedule_none(self):
-        """Test that None schedule returns False."""
-        result = time_handler.is_within_schedule(None)
-        self.assertFalse(result)
-
-    def test_schedule_empty_dict(self):
-        """Test that empty dict returns False."""
-        result = time_handler.is_within_schedule({})
-        self.assertFalse(result)
-
-    def test_schedule_enabled_but_day_disabled(self):
-        """Test schedule enabled but current day disabled."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        current_day = now.format('dddd').lower()
-
-        time_schedule_limit = {
-            'enabled': True,
-            'timezone': timezone_str,
-            current_day: {
-                'enabled': False,
-                'start_time': '09:00',
-                'duration': {'hours': 8, 'minutes': 0}
-            }
-        }
-
-        result = time_handler.is_within_schedule(time_schedule_limit)
-        self.assertFalse(result, "Disabled day should return False")
-
-    def test_schedule_enabled_within_time(self):
-        """Test schedule enabled and within time window."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        current_day = now.format('dddd').lower()
-        current_hour = now.format('HH:00')
-
-        time_schedule_limit = {
-            'enabled': True,
-            'timezone': timezone_str,
-            current_day: {
-                'enabled': True,
-                'start_time': current_hour,
-                'duration': {'hours': 2, 'minutes': 0}
-            }
-        }
-
-        result = time_handler.is_within_schedule(time_schedule_limit)
-        self.assertTrue(result, "Current time should be within schedule")
-
-    def test_schedule_enabled_outside_time(self):
-        """Test schedule enabled but outside time window."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        current_day = now.format('dddd').lower()
-        # Set time to 3 hours ago
-        past_time = now.shift(hours=-3).format('HH:mm')
-
-        time_schedule_limit = {
-            'enabled': True,
-            'timezone': timezone_str,
-            current_day: {
-                'enabled': True,
-                'start_time': past_time,
-                'duration': {'hours': 1, 'minutes': 0}  # Only 1 hour duration
-            }
-        }
-
-        result = time_handler.is_within_schedule(time_schedule_limit)
-        self.assertFalse(result, "3 hours ago with 1 hour duration should be False")
-
-    def test_schedule_with_default_timezone(self):
-        """Test schedule without timezone uses default."""
-        now = arrow.now('America/New_York')
-        current_day = now.format('dddd').lower()
-        current_hour = now.format('HH:00')
-
-        time_schedule_limit = {
-            'enabled': True,
-            # No timezone specified
-            current_day: {
-                'enabled': True,
-                'start_time': current_hour,
-                'duration': {'hours': 2, 'minutes': 0}
-            }
-        }
-
-        # Should use default UTC, but since we're testing with NY time,
-        # the result depends on time difference
-        result = time_handler.is_within_schedule(
-            time_schedule_limit,
-            default_tz='America/New_York'
-        )
-        self.assertTrue(result, "Should work with default timezone")
-
-    def test_schedule_different_timezones(self):
-        """Test schedule works correctly across different timezones."""
-        # Test with Tokyo timezone
-        timezone_str = 'Asia/Tokyo'
-        now = arrow.now(timezone_str)
-        current_day = now.format('dddd').lower()
-        current_hour = now.format('HH:00')
-
-        time_schedule_limit = {
-            'enabled': True,
-            'timezone': timezone_str,
-            current_day: {
-                'enabled': True,
-                'start_time': current_hour,
-                'duration': {'hours': 1, 'minutes': 30}
-            }
-        }
-
-        result = time_handler.is_within_schedule(time_schedule_limit)
-        self.assertTrue(result)
-
-    def test_schedule_with_minutes_in_duration(self):
-        """Test schedule with minutes specified in duration."""
-        timezone_str = 'UTC'
-        now = arrow.now(timezone_str)
-        current_day = now.format('dddd').lower()
-        current_time = now.format('HH:mm')
-
-        time_schedule_limit = {
-            'enabled': True,
-            'timezone': timezone_str,
-            current_day: {
-                'enabled': True,
-                'start_time': current_time,
-                'duration': {'hours': 0, 'minutes': 45}
-            }
-        }
-
-        result = time_handler.is_within_schedule(time_schedule_limit)
-        self.assertTrue(result, "Should handle minutes in duration")
-
-    def test_schedule_with_timezone_whitespace(self):
-        """Test that timezone with whitespace is handled."""
-        timezone_str = '  UTC  '
-        now = arrow.now('UTC')
-        current_day = now.format('dddd').lower()
-        current_hour = now.format('HH:00')
-
-        time_schedule_limit = {
-            'enabled': True,
-            'timezone': timezone_str,
-            current_day: {
-                'enabled': True,
-                'start_time': current_hour,
-                'duration': {'hours': 1, 'minutes': 0}
-            }
-        }
-
-        result = time_handler.is_within_schedule(time_schedule_limit)
-        self.assertTrue(result, "Should handle timezone with whitespace")
-
-
-class TestWeekdayEnum(unittest.TestCase):
-    """Tests for the Weekday enum."""
-
-    def test_weekday_values(self):
-        """Test that weekday enum has correct values."""
-        self.assertEqual(time_handler.Weekday.Monday, 0)
-        self.assertEqual(time_handler.Weekday.Tuesday, 1)
-        self.assertEqual(time_handler.Weekday.Wednesday, 2)
-        self.assertEqual(time_handler.Weekday.Thursday, 3)
-        self.assertEqual(time_handler.Weekday.Friday, 4)
-        self.assertEqual(time_handler.Weekday.Saturday, 5)
-        self.assertEqual(time_handler.Weekday.Sunday, 6)
-
-    def test_weekday_string_access(self):
-        """Test accessing weekday enum by string."""
-        self.assertEqual(time_handler.Weekday['Monday'], 0)
-        self.assertEqual(time_handler.Weekday['Sunday'], 6)
-
-
-if __name__ == '__main__':
-    unittest.main()

changedetectionio/tests/util.py

@@ -188,10 +188,6 @@ def new_live_server_setup(live_server):
         ctype = request.args.get('content_type')
         status_code = request.args.get('status_code')
         content = request.args.get('content') or None
-        delay = int(request.args.get('delay', 0))
-
-        if delay:
-            time.sleep(delay)
 
         # Used to just try to break the header detection
         uppercase_headers = request.args.get('uppercase_headers')

changedetectionio/time_handler.py

@@ -1,5 +1,6 @@
-import arrow
+from datetime import timedelta, datetime
 from enum import IntEnum
+from zoneinfo import ZoneInfo
 
 
 class Weekday(IntEnum):
@@ -39,65 +40,54 @@ def am_i_inside_time(
 
     # Parse the start time
     try:
-        hour, minute = map(int, time_str.split(':'))
-        if not (0 <= hour <= 23 and 0 <= minute <= 59):
-            raise ValueError
-    except (ValueError, AttributeError):
+        target_time = datetime.strptime(time_str, '%H:%M').time()
+    except ValueError:
         raise ValueError(f"Invalid time_str: '{time_str}'. Must be in 'HH:MM' format.")
 
-    # Get the current time in the specified timezone
+    # Define the timezone
     try:
-        now_tz = arrow.now(timezone_str.strip())
-    except Exception as e:
+        tz = ZoneInfo(timezone_str)
+    except Exception:
         raise ValueError(f"Invalid timezone_str: '{timezone_str}'. Must be a valid timezone identifier.")
 
+    # Get the current time in the specified timezone
+    now_tz = datetime.now(tz)
+
     # Check if the current day matches the target day or overlaps due to duration
     current_weekday = now_tz.weekday()
-    # Create start datetime for today in target timezone
-    start_datetime_tz = now_tz.replace(hour=hour, minute=minute, second=0, microsecond=0)
+    start_datetime_tz = datetime.combine(now_tz.date(), target_time, tzinfo=tz)
 
     # Handle previous day's overlap
     if target_weekday == (current_weekday - 1) % 7:
         # Calculate start and end times for the overlap from the previous day
-        start_datetime_tz = start_datetime_tz.shift(days=-1)
-        end_datetime_tz = start_datetime_tz.shift(minutes=duration)
+        start_datetime_tz -= timedelta(days=1)
+        end_datetime_tz = start_datetime_tz + timedelta(minutes=duration)
         if start_datetime_tz <= now_tz < end_datetime_tz:
             return True
 
     # Handle current day's range
     if target_weekday == current_weekday:
-        end_datetime_tz = start_datetime_tz.shift(minutes=duration)
+        end_datetime_tz = start_datetime_tz + timedelta(minutes=duration)
         if start_datetime_tz <= now_tz < end_datetime_tz:
             return True
 
     # Handle next day's overlap
     if target_weekday == (current_weekday + 1) % 7:
-        end_datetime_tz = start_datetime_tz.shift(minutes=duration)
-        if now_tz < start_datetime_tz and now_tz.shift(days=1) < end_datetime_tz:
+        end_datetime_tz = start_datetime_tz + timedelta(minutes=duration)
+        if now_tz < start_datetime_tz and now_tz + timedelta(days=1) < end_datetime_tz:
             return True
 
     return False
 
 
 def is_within_schedule(time_schedule_limit, default_tz="UTC"):
-    """
-    Check if the current time is within a scheduled time window.
-
-    Parameters:
-        time_schedule_limit (dict): Schedule configuration with timezone, day settings, etc.
-        default_tz (str): Default timezone to use if not specified. Default is 'UTC'.
-
-    Returns:
-        bool: True if current time is within the schedule, False otherwise.
-    """
     if time_schedule_limit and time_schedule_limit.get('enabled'):
         # Get the timezone the time schedule is in, so we know what day it is there
         tz_name = time_schedule_limit.get('timezone')
         if not tz_name:
             tz_name = default_tz
 
-        # Get current day name in the target timezone
-        now_day_name_in_tz = arrow.now(tz_name.strip()).format('dddd')
+        now_day_name_in_tz = datetime.now(ZoneInfo(tz_name.strip())).strftime('%A')
         selected_day_schedule = time_schedule_limit.get(now_day_name_in_tz.lower())
         if not selected_day_schedule.get('enabled'):
             return False

requirements.txt

@@ -1,6 +1,5 @@
 # eventlet>=0.38.0  # Removed - replaced with threading mode for better Python 3.12+ compatibility
 feedgen~=0.9
-feedparser~=6.0  # For parsing RSS/Atom feeds
 flask-compress
 # 0.6.3 included compatibility fix for werkzeug 3.x (2.x had deprecation of url handlers)
 flask-login>=0.6.3
@@ -10,14 +9,14 @@ flask_restful
 flask_cors # For the Chrome extension to operate
 janus # Thread-safe async/sync queue bridge
 flask_wtf~=1.2
-flask~=3.1
+flask~=2.3
 flask-socketio~=5.5.1
 python-socketio~=5.13.0
 python-engineio~=4.12.3
 inscriptis~=2.2
 pytz
 timeago~=1.0
-validators~=0.35
+validators~=0.21
 
 
 # Set these versions together to avoid a RequestsDependencyWarning
@@ -42,9 +41,6 @@ jsonpath-ng~=1.5.3
 # Notification library
 apprise==1.9.5
 
-# Lightweight URL linkifier for notifications
-linkify-it-py
-
 # - Needed for apprise/spush, and maybe others? hopefully doesnt trigger a rust compile.
 # - Requires extra wheel for rPi, adds build time for arm/v8 which is not in piwheels
 # Pinned to 43.0.1 for ARM compatibility (45.x may not have pre-built ARM wheels)
@@ -56,7 +52,7 @@ cryptography==44.0.1
 paho-mqtt!=2.0.*
 
 # Used for CSS filtering, JSON extraction from HTML
-beautifulsoup4>=4.0.0,<=4.14.2
+beautifulsoup4>=4.0.0,<=4.13.5
 
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
 # #2328 - 5.2.0 and 5.2.1 had extra CPU flag CFLAGS set which was not compatible on older hardware
@@ -66,13 +62,17 @@ lxml >=4.8.0,<6,!=5.2.0,!=5.2.1
 
 # XPath 2.0-3.1 support - 4.2.0 had issues, 4.1.5 stable
 # Consider updating to latest stable version periodically
-elementpath==5.0.4
+elementpath==4.1.5
 
 selenium~=4.31.0
 
+# https://github.com/pallets/werkzeug/issues/2985
+# Maybe related to pytest?
+werkzeug==3.0.6
+
 # Templating, so far just in the URLs but in the future can be for the notifications also
 jinja2~=3.1
-arrow
+jinja2-time
 openpyxl
 # https://peps.python.org/pep-0508/#environment-markers
 # https://github.com/dgtlmoon/changedetection.io/pull/1009
@@ -88,7 +88,6 @@ pyppeteerstealth>=0.0.4
 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup
 pytest ~=7.2
 pytest-flask ~=1.2
-pytest-mock ~=3.15
 
 # Anything 4.0 and up but not 5.0
 jsonschema ~= 4.0
@@ -125,9 +124,8 @@ price-parser
 
 # flask_socket_io - incorrect package name, already have flask-socketio above
 
-# Lightweight MIME type detection (saves ~14MB memory vs python-magic/libmagic)
-# Used for detecting correct favicon type and content-type detection
-puremagic
+# So far for detecting correct favicon type, but for other things in the future
+python-magic
 
 # Scheduler - Windows seemed to miss a lot of default timezone info (even "UTC" !)
 tzdata

setup.py

@@ -5,8 +5,6 @@ import re
 import sys
 
 from setuptools import setup, find_packages
-from setuptools.command.build_py import build_py
-import shutil
 
 here = os.path.abspath(os.path.dirname(__file__))
 
@@ -24,20 +22,6 @@ def find_version(*file_paths):
     raise RuntimeError("Unable to find version string.")
 
 
-class BuildPyCommand(build_py):
-    """Custom build command to copy api-spec.yaml to the package."""
-    def run(self):
-        build_py.run(self)
-        # Ensure the docs directory exists in the build output
-        docs_dir = os.path.join(self.build_lib, 'changedetectionio', 'docs')
-        os.makedirs(docs_dir, exist_ok=True)
-        # Copy api-spec.yaml to the package
-        shutil.copy(
-            os.path.join(here, 'docs', 'api-spec.yaml'),
-            os.path.join(docs_dir, 'api-spec.yaml')
-        )
-
-
 install_requires = open('requirements.txt').readlines()
 
 setup(
@@ -53,10 +37,9 @@ setup(
     scripts=["changedetection.py"],
     author='dgtlmoon',
     url='https://changedetection.io',
-    packages=find_packages(include=['changedetectionio', 'changedetectionio.*']),
+    packages=['changedetectionio'],
     include_package_data=True,
     install_requires=install_requires,
-    cmdclass={'build_py': BuildPyCommand},
     license="Apache License 2.0",
     python_requires=">= 3.10",
     classifiers=['Intended Audience :: Customer Service',