Closes #3367

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.50.35'
+__version__ = '0.50.38'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/api/api_schema.py

@@ -96,7 +96,10 @@ def build_watch_json_schema(d):
                                                            "enum": ["html_requests", "html_webdriver"]
                                                            })
 
-
+    schema['properties']['processor'] = {"anyOf": [
+        {"type": "string", "enum": ["restock_diff", "text_json_diff"]},
+        {"type": "null"}
+    ]}
 
     # All headers must be key/value type dict
     schema['properties']['headers'] = {

changedetectionio/blueprint/ui/notification.py

@@ -2,7 +2,7 @@ from flask import Blueprint, request, make_response
 import random
 from loguru import logger
 
-from changedetectionio.notification_service import NotificationContextData
+from changedetectionio.notification_service import NotificationContextData, set_basic_notification_vars
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.auth_decorator import login_optionally_required
 
@@ -95,7 +95,44 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 n_object['notification_body'] = "Test body"
 
             n_object['as_async'] = False
-            n_object.update(watch.extra_notification_token_values())
+
+            #  Same like in notification service, should be refactored
+            dates = []
+            trigger_text = ''
+            snapshot_contents = ''
+            if watch:
+                watch_history = watch.history
+                dates = list(watch_history.keys())
+                trigger_text = watch.get('trigger_text', [])
+                # Add text that was triggered
+                if len(dates):
+                    snapshot_contents = watch.get_history_snapshot(dates[-1])
+                else:
+                    snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
+
+                if len(trigger_text):
+                    from . import html_tools
+                    triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
+                    if triggered_text:
+                        triggered_text = '\n'.join(triggered_text)
+
+            # Could be called as a 'test notification' with only 1 snapshot available
+            prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
+            current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
+
+
+
+            if len(dates) > 1:
+                prev_snapshot = watch.get_history_snapshot(dates[-2])
+                current_snapshot = watch.get_history_snapshot(dates[-1])
+
+            n_object.update(set_basic_notification_vars(snapshot_contents=snapshot_contents,
+                                                        current_snapshot=current_snapshot,
+                                                        prev_snapshot=prev_snapshot,
+                                                        watch=watch,
+                                                        triggered_text=trigger_text))
+
+
             sent_obj = process_notification(n_object, datastore)
 
         except Exception as e:

changedetectionio/notification/handler.py

@@ -5,13 +5,15 @@ from apprise import NotifyFormat
 from loguru import logger
 from urllib.parse import urlparse
 from .apprise_plugin.assets import apprise_asset, APPRISE_AVATAR_URL
-from .apprise_plugin.custom_handlers import SUPPORTED_HTTP_METHODS
 from .email_helpers import as_monospaced_html_email
 from ..diff import HTML_REMOVED_STYLE, REMOVED_PLACEMARKER_OPEN, REMOVED_PLACEMARKER_CLOSED, ADDED_PLACEMARKER_OPEN, HTML_ADDED_STYLE, \
     ADDED_PLACEMARKER_CLOSED, CHANGED_INTO_PLACEMARKER_OPEN, CHANGED_INTO_PLACEMARKER_CLOSED, CHANGED_PLACEMARKER_OPEN, \
     CHANGED_PLACEMARKER_CLOSED, HTML_CHANGED_STYLE, HTML_CHANGED_INTO_STYLE
-from ..notification_service import NotificationContextData, CUSTOM_LINEBREAK_PLACEHOLDER
+import re
 
+from ..notification_service import NotificationContextData
+
+newline_re = re.compile(r'\r\n|\r|\n')
 
 
 def markup_text_links_to_html(body):
@@ -127,6 +129,62 @@ def apply_standard_markdown_to_body(n_body):
     return n_body
 
 
+def replace_placemarkers_in_text(text, url, requested_output_format):
+    """
+    Replace diff placemarkers in text based on the URL service type and requested output format.
+    Used for both notification title and body to ensure consistent placeholder replacement.
+
+    :param text: The text to process
+    :param url: The notification URL (to detect service type)
+    :param requested_output_format: The output format (html, htmlcolor, markdown, text, etc.)
+    :return: Processed text with placemarkers replaced
+    """
+    if not text:
+        return text
+
+    if url.startswith('tgram://'):
+        # Telegram only supports a limited subset of HTML
+        # Use strikethrough for removed content, bold for added content
+        text = text.replace(REMOVED_PLACEMARKER_OPEN, '<s>')
+        text = text.replace(REMOVED_PLACEMARKER_CLOSED, '</s>')
+        text = text.replace(ADDED_PLACEMARKER_OPEN, '<b>')
+        text = text.replace(ADDED_PLACEMARKER_CLOSED, '</b>')
+        # Handle changed/replaced lines (old → new)
+        text = text.replace(CHANGED_PLACEMARKER_OPEN, '<s>')
+        text = text.replace(CHANGED_PLACEMARKER_CLOSED, '</s>')
+        text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, '<b>')
+        text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, '</b>')
+    elif (url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks')
+          or url.startswith('https://discord.com/api')) and requested_output_format == 'html':
+        # Discord doesn't support HTML, use Discord markdown
+        text = apply_discord_markdown_to_body(n_body=text)
+    elif requested_output_format == 'htmlcolor':
+        # https://github.com/dgtlmoon/changedetection.io/issues/821#issuecomment-1241837050
+        text = text.replace(REMOVED_PLACEMARKER_OPEN, f'<span style="{HTML_REMOVED_STYLE}" role="deletion" aria-label="Removed text" title="Removed text">')
+        text = text.replace(REMOVED_PLACEMARKER_CLOSED, f'</span>')
+        text = text.replace(ADDED_PLACEMARKER_OPEN, f'<span style="{HTML_ADDED_STYLE}" role="insertion" aria-label="Added text" title="Added text">')
+        text = text.replace(ADDED_PLACEMARKER_CLOSED, f'</span>')
+        # Handle changed/replaced lines (old → new)
+        text = text.replace(CHANGED_PLACEMARKER_OPEN, f'<span style="{HTML_CHANGED_STYLE}" role="note" aria-label="Changed text" title="Changed text">')
+        text = text.replace(CHANGED_PLACEMARKER_CLOSED, f'</span>')
+        text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'<span style="{HTML_CHANGED_INTO_STYLE}" role="note" aria-label="Changed into" title="Changed into">')
+        text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'</span>')
+    elif requested_output_format == 'markdown':
+        # Markdown to HTML - Apprise will convert this to HTML
+        text = apply_standard_markdown_to_body(n_body=text)
+    else:
+        # plaintext, html, and default - use simple text markers
+        text = text.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ')
+        text = text.replace(REMOVED_PLACEMARKER_CLOSED, '')
+        text = text.replace(ADDED_PLACEMARKER_OPEN, '(added) ')
+        text = text.replace(ADDED_PLACEMARKER_CLOSED, '')
+        text = text.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ')
+        text = text.replace(CHANGED_PLACEMARKER_CLOSED, f'')
+        text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ')
+        text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')
+
+    return text
+
 def apply_service_tweaks(url, n_body, n_title, requested_output_format):
 
     # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
@@ -138,6 +196,12 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format):
     if not n_body or not n_body.strip():
         return url, n_body, n_title
 
+    # Normalize URL scheme to lowercase to prevent case-sensitivity issues
+    # e.g., "Discord://webhook" -> "discord://webhook", "TGRAM://bot123" -> "tgram://bot123"
+    scheme_separator_pos = url.find('://')
+    if scheme_separator_pos > 0:
+        url = url[:scheme_separator_pos].lower() + url[scheme_separator_pos:]
+
     # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
     parsed = urlparse(url)
     k = '?' if not parsed.query else '&'
@@ -149,24 +213,22 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format):
             and not url.startswith('put'):
         url += k + f"avatar_url={APPRISE_AVATAR_URL}"
 
+    # Replace placemarkers in title first (this was the missing piece causing the bug)
+    # Titles are ALWAYS plain text across all notification services (Discord embeds, Slack attachments,
+    # email Subject headers, etc.), so we always use 'text' format for title placemarker replacement
+    # Looking over apprise library it seems that all plugins only expect plain-text.
+    n_title = replace_placemarkers_in_text(n_title, url, 'text')
+
     if url.startswith('tgram://'):
         # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
         # re https://github.com/dgtlmoon/changedetection.io/issues/555
         # @todo re-use an existing library we have already imported to strip all non-allowed tags
         n_body = n_body.replace('<br>', '\n')
         n_body = n_body.replace('</br>', '\n')
-        n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\n')
+        n_body = newline_re.sub('\n', n_body)
 
-        # Use strikethrough for removed content, bold for added content
+        # Replace placemarkers for body
-        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '<s>')
+        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
-        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '</s>')
-        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '<b>')
-        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '</b>')
-        # Handle changed/replaced lines (old → new)
-        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, '<s>')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, '</s>')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, '<b>')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, '</b>')
 
         # real limit is 4096, but minus some for extra metadata
         payload_max_size = 3600
@@ -180,7 +242,7 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format):
         # Discord doesn't support HTML, replace <br> with newlines
         n_body = n_body.strip().replace('<br>', '\n')
         n_body = n_body.replace('</br>', '\n')
-        n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\n')
+        n_body = newline_re.sub('\n', n_body)
 
         # Don't replace placeholders or truncate here - let the custom Discord plugin handle it
         # The plugin will use embeds (6000 char limit across all embeds) if placeholders are present,
@@ -190,7 +252,7 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format):
         if requested_output_format == 'html':
             # No diff placeholders, use Discord markdown for any other formatting
             # Use Discord markdown: strikethrough for removed, bold for added
-            n_body = apply_discord_markdown_to_body(n_body=n_body)
+            n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
 
             # Apply 2000 char limit for plain content
             payload_max_size = 1700
@@ -201,40 +263,17 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format):
 
     # Is not discord/tgram and they want htmlcolor
     elif requested_output_format == 'htmlcolor':
-        # https://github.com/dgtlmoon/changedetection.io/issues/821#issuecomment-1241837050
+        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
-        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, f'<span style="{HTML_REMOVED_STYLE}" role="deletion" aria-label="Removed text" title="Removed text">')
+        n_body = newline_re.sub('<br>\n', n_body)
-        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, f'</span>')
-        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, f'<span style="{HTML_ADDED_STYLE}" role="insertion" aria-label="Added text" title="Added text">')
-        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, f'</span>')
-        # Handle changed/replaced lines (old → new)
-        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'<span style="{HTML_CHANGED_STYLE}" role="note" aria-label="Changed text" title="Changed text">')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'</span>')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'<span style="{HTML_CHANGED_INTO_STYLE}" role="note" aria-label="Changed into" title="Changed into">')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'</span>')
-        n_body = n_body.replace('\n', f'{CUSTOM_LINEBREAK_PLACEHOLDER}\n')
     elif requested_output_format == 'html':
-        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ')
+        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
-        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '')
+        n_body = newline_re.sub('<br>\n', n_body)
-        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '(added) ')
-        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')
-        n_body = n_body.replace('\n', f'{CUSTOM_LINEBREAK_PLACEHOLDER}\n')
     elif requested_output_format == 'markdown':
         # Markdown to HTML - Apprise will convert this to HTML
-        n_body = apply_standard_markdown_to_body(n_body=n_body)
+        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
 
     else: #plaintext etc default
-        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ')
+        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
-        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '')
-        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '(added) ')
-        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')
 
     return url, n_body, n_title
 
@@ -295,24 +334,18 @@ def process_notification(n_object: NotificationContextData, datastore):
     with (apprise.LogCapture(level=apprise.logging.DEBUG) as logs):
         for url in n_object['notification_urls']:
 
-            # Get the notification body from datastore
             n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
+            n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
 
             if n_object.get('markup_text_links_to_html_links'):
                 n_body = markup_text_links_to_html(body=n_body)
 
-            n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
-
             url = url.strip()
-            if url.startswith('#'):
+            if not url or url.startswith('#'):
-                logger.trace(f"Skipping commented out notification URL - {url}")
+                logger.debug(f"Skipping commented out or empty notification URL - '{url}'")
                 continue
 
-            if not url:
+            logger.info(f">> Process Notification: AppRise start notifying '{url}'")
-                logger.warning(f"Process Notification: skipping empty notification URL.")
-                continue
-
-            logger.info(f">> Process Notification: AppRise notifying {url}")
             url = jinja_render(template_str=url, **notification_parameters)
 
             # If it's a plaintext document, and they want HTML type email/alerts, so it needs to be escaped
@@ -353,25 +386,18 @@ def process_notification(n_object: NotificationContextData, datastore):
                     requested_output_format = NotifyFormat.HTML.value
                     apprise_input_format = NotifyFormat.HTML.value  # Changed from MARKDOWN to HTML
 
-                # Could have arrived at any stage, so we dont end up running .escape on it
-                if 'html' in requested_output_format:
-                    n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '<br>\r\n')
-                else:
-                    # texty types
-                    n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\r\n')
-
             else:
                 # ?format was IN the apprise URL, they are kind of on their own here, we will try our best
                 if 'format=html' in url:
-                    n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '<br>\r\n')
+                    n_body = newline_re.sub('<br>\r\n', n_body)
                     # This will also prevent apprise from doing conversion
                     apprise_input_format = NotifyFormat.HTML.value
                     requested_output_format = NotifyFormat.HTML.value
                 elif 'format=text' in url:
-                    n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\r\n')
                     apprise_input_format = NotifyFormat.TEXT.value
                     requested_output_format = NotifyFormat.TEXT.value
 
+
             sent_objs.append({'title': n_title,
                               'body': n_body,
                               'url': url})

changedetectionio/notification_service.py

@@ -9,11 +9,8 @@ for both sync and async workers
 from loguru import logger
 import time
 
-from changedetectionio.model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
 from changedetectionio.notification import default_notification_format, valid_notification_formats
 
-# This gets modified on notification time (handler.py) depending on the required notification output
-CUSTOM_LINEBREAK_PLACEHOLDER='@BR@'
 
 
 # What is passed around as notification context, also used as the complete list of valid {{ tokens }}
@@ -23,10 +20,14 @@ class NotificationContextData(dict):
             'base_url': None,
             'current_snapshot': None,
             'diff': None,
+            'diff_clean': None,
             'diff_added': None,
+            'diff_added_clean': None,
             'diff_full': None,
+            'diff_full_clean': None,
             'diff_patch': None,
             'diff_removed': None,
+            'diff_removed_clean': None,
             'diff_url': None,
             'markup_text_links_to_html_links': False, # If automatic conversion of plaintext to HTML should happen
             'notification_timestamp': time.time(),
@@ -71,6 +72,38 @@ class NotificationContextData(dict):
 
         super().__setitem__(key, value)
 
+
+def set_basic_notification_vars(snapshot_contents, current_snapshot, prev_snapshot, watch, triggered_text):
+    now = time.time()
+    from changedetectionio import diff
+
+    n_object = {
+        'current_snapshot': snapshot_contents,
+        'diff': diff.render_diff(prev_snapshot, current_snapshot),
+        'diff_clean': diff.render_diff(prev_snapshot, current_snapshot, include_change_type_prefix=False),
+        'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False),
+        'diff_added_clean': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, include_change_type_prefix=False),
+        'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True),
+        'diff_full_clean': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, include_change_type_prefix=False),
+        'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, patch_format=True),
+        'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False),
+        'diff_removed_clean': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, include_change_type_prefix=False),
+        'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
+        'triggered_text': triggered_text,
+        'uuid': watch.get('uuid') if watch else None,
+        'watch_url': watch.get('url') if watch else None,
+        'watch_uuid': watch.get('uuid') if watch else None,
+        'watch_mime_type': watch.get('content-type')
+    }
+
+    # The \n's in the content from the above will get converted to <br> etc depending on the notification format
+
+    if watch:
+        n_object.update(watch.extra_notification_token_values())
+
+    logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time() - now:.3f}s")
+    return n_object
+
 class NotificationService:
     """
     Standalone notification service that handles all notification functionality
@@ -85,7 +118,6 @@ class NotificationService:
         """
         Queue a notification for a watch with full diff rendering and template variables
         """
-        from changedetectionio import diff
         from changedetectionio.notification import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
 
         if not isinstance(n_object, NotificationContextData):
@@ -94,8 +126,6 @@ class NotificationService:
         dates = []
         trigger_text = ''
 
-        now = time.time()
-
         if watch:
             watch_history = watch.history
             dates = list(watch_history.keys())
@@ -117,7 +147,7 @@ class NotificationService:
             from . import html_tools
             triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
             if triggered_text:
-                triggered_text = CUSTOM_LINEBREAK_PLACEHOLDER.join(triggered_text)
+                triggered_text = '\n'.join(triggered_text)
 
         # Could be called as a 'test notification' with only 1 snapshot available
         prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
@@ -127,25 +157,13 @@ class NotificationService:
             prev_snapshot = watch.get_history_snapshot(dates[-2])
             current_snapshot = watch.get_history_snapshot(dates[-1])
 
-        n_object.update({
-            'current_snapshot': snapshot_contents,
-            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER),
-            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER),
-            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER),
-            'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER, patch_format=True),
-            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER),
-            'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
-            'triggered_text': triggered_text,
-            'uuid': watch.get('uuid') if watch else None,
-            'watch_url': watch.get('url') if watch else None,
-            'watch_uuid': watch.get('uuid') if watch else None,
-            'watch_mime_type': watch.get('content-type')
-        })
 
-        if watch:
+        n_object.update(set_basic_notification_vars(snapshot_contents=snapshot_contents,
-            n_object.update(watch.extra_notification_token_values())
+                                                    current_snapshot=current_snapshot,
+                                                    prev_snapshot=prev_snapshot,
+                                                    watch=watch,
+                                                    triggered_text=triggered_text))
 
-        logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
         logger.debug("Queued notification for sending")
         self.notification_q.put(n_object)
 

changedetectionio/static/styles/scss/styles.scss

@@ -329,12 +329,18 @@ a.pure-button-selected {
 .notifications-wrapper {
   padding-top: 0.5rem;
   #notification-test-log {
-    padding-top: 1rem;
+    margin-top: 1rem;
+    padding: 1rem;
     white-space: pre-wrap;
     word-break: break-word;
     overflow-wrap: break-word;
     max-width: 100%;
     box-sizing: border-box;
+    max-height: 12rem;
+    overflow-y: scroll;
+    border: 1px solid var(--color-border-notification);
+    border-radius: 5px;
+
   }
 }
 

changedetectionio/templates/_common_fields.html

@@ -87,19 +87,35 @@
 									<tr>
                                         <td><code>{{ '{{diff}}' }}</code></td>
                                         <td>The diff output - only changes, additions, and removals</td>
+                                    </tr>
+									<tr>
+                                        <td><code>{{ '{{diff_clean}}' }}</code></td>
+                                        <td>The diff output - only changes, additions, and removals &dash; <i>Without (added) prefix or colors</i></td>
                                     </tr>
 									<tr>
                                         <td><code>{{ '{{diff_added}}' }}</code></td>
                                         <td>The diff output - only changes and additions</td>
+                                    </tr>
+									<tr>
+                                        <td><code>{{ '{{diff_added_clean}}' }}</code></td>
+                                        <td>The diff output - only changes and additions &dash; <i>Without (added) prefix or colors</i></td>
                                     </tr>
 									<tr>
                                         <td><code>{{ '{{diff_removed}}' }}</code></td>
                                         <td>The diff output - only changes and removals</td>
+                                    </tr>
+									<tr>
+                                        <td><code>{{ '{{diff_removed_clean}}' }}</code></td>
+                                        <td>The diff output - only changes and removals &dash; <i>Without (added) prefix or colors</i></td>
                                     </tr>
                                     <tr>
                                         <td><code>{{ '{{diff_full}}' }}</code></td>
                                         <td>The diff output - full difference output</td>
                                     </tr>
+                                    <tr>
+                                        <td><code>{{ '{{diff_full_clean}}' }}</code></td>
+                                        <td>The diff output - full difference output &dash; <i>Without (added) prefix or colors</i></td>
+                                    </tr>
                                     <tr>
                                         <td><code>{{ '{{diff_patch}}' }}</code></td>
                                         <td>The diff output - patch in unified format</td>

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -3,8 +3,9 @@ from flask import url_for
 from email import message_from_string
 from email.policy import default as email_policy
 
-from changedetectionio.diff import HTML_REMOVED_STYLE, HTML_ADDED_STYLE, HTML_CHANGED_STYLE
+from changedetectionio.diff import HTML_REMOVED_STYLE, HTML_ADDED_STYLE, HTML_CHANGED_STYLE, REMOVED_PLACEMARKER_OPEN, \
-from changedetectionio.notification_service import NotificationContextData, CUSTOM_LINEBREAK_PLACEHOLDER
+    CHANGED_PLACEMARKER_OPEN, ADDED_PLACEMARKER_OPEN
+from changedetectionio.notification_service import NotificationContextData
 from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
     wait_for_all_checks, \
     set_longer_modified_response, delete_all_watches
@@ -100,7 +101,6 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     text_content = text_part.get_content()
     assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
     assert 'fallback-body\r\n' in text_content  # The plaintext part
-    assert CUSTOM_LINEBREAK_PLACEHOLDER not in text_content
 
     # Second part should be text/html
     html_part = parts[1]
@@ -109,7 +109,6 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     assert 'some text<br>' in html_content  # We converted \n from the notification body
     assert 'fallback-body<br>' in html_content  # kept the original <br>
     assert '(added) So let\'s see what happens.<br>' in html_content  # the html part
-    assert CUSTOM_LINEBREAK_PLACEHOLDER not in html_content
     delete_all_watches(client)
 
 
@@ -124,8 +123,8 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
     res = client.post(
         url_for("settings.settings_page"),
         data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_title": "fallback-title {{watch_title}}  {{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}  " + default_notification_title,
-              "application-notification_body": "some text\n" + default_notification_body,
+              "application-notification_body": f"some text\n" + default_notification_body + f"\nMore output test\n{ALL_MARKUP_TOKENS}",
               "application-notification_format": 'text',
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
@@ -148,9 +147,17 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
 
     msg_raw = get_last_message_from_smtp_server()
     assert len(msg_raw) >= 1
-
+    #time.sleep(60)
     # Parse the email properly using Python's email library
     msg = message_from_string(msg_raw, policy=email_policy)
+    # Subject/title got marked up
+    subject = msg['subject']
+    # Subject should always be plaintext and never marked up to anything else
+    assert REMOVED_PLACEMARKER_OPEN not in subject
+    assert CHANGED_PLACEMARKER_OPEN not in subject
+    assert ADDED_PLACEMARKER_OPEN not in subject
+    assert 'diff added didnt split' not in subject
+    assert '(changed) Which is across' in subject
 
     # The email should be plain text only (not multipart)
     assert not msg.is_multipart()
@@ -177,7 +184,7 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
     res = client.post(
         url_for("settings.settings_page"),
         data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_title": "fallback-title {{watch_title}} - diff_added_lines_test : '{{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}' " + default_notification_title,
               "application-notification_body": f"some text\n{default_notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}",
               "application-notification_format": 'htmlcolor',
               "requests-time_between_check-minutes": 180,
@@ -211,6 +218,18 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
 
     # Parse the email properly using Python's email library
     msg = message_from_string(msg_raw, policy=email_policy)
+    # Subject/title got marked up
+    subject = msg['subject']
+    # Subject should always be plaintext and never marked up to anything else
+    assert REMOVED_PLACEMARKER_OPEN not in subject
+    assert CHANGED_PLACEMARKER_OPEN not in subject
+    assert ADDED_PLACEMARKER_OPEN not in subject
+    assert 'diff added didnt split' not in subject
+    assert '(changed) Which is across' in subject
+    assert 'head title' in subject
+    assert "span" not in subject
+    assert 'background-color' not in subject
+
 
     # The email should have two bodies (multipart/alternative with text/plain and text/html)
     assert msg.is_multipart()
@@ -249,7 +268,7 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_
     res = client.post(
         url_for("settings.settings_page"),
         data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_title": "fallback-title  diff_added_lines_test : '{{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}' " + default_notification_title,
               "application-notification_body": "*header*\n\nsome text\n" + default_notification_body,
               "application-notification_format": 'markdown',
               "requests-time_between_check-minutes": 180,
@@ -287,6 +306,14 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_
     # The email should have two bodies (multipart/alternative with text/plain and text/html)
     assert msg.is_multipart()
     assert msg.get_content_type() == 'multipart/alternative'
+    subject = msg['subject']
+    # Subject should always be plaintext and never marked up to anything else
+    assert REMOVED_PLACEMARKER_OPEN not in subject
+    assert CHANGED_PLACEMARKER_OPEN not in subject
+    assert ADDED_PLACEMARKER_OPEN not in subject
+    assert 'diff added didnt split' not in subject
+    assert '(changed) Which is across' in subject
+
 
     # Get the parts
     parts = list(msg.iter_parts())
@@ -305,7 +332,10 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_
     assert html_part.get_content_type() == 'text/html'
     html_content = html_part.get_content()
     assert '<p><em>header</em></p>' in html_content
-    assert '<strong>So let\'s see what happens.</strong><br>' in html_content # Additions are <strong> in markdown
+    assert '<strong>So let\'s see what happens.</strong><br />' in html_content # Additions are <strong> in markdown
+    # the '<br />' will come from apprises conversion, not from our code, we would rather use '<br>' correctly
+    # the '<br />' is actually a nice way to know if apprise done the conversion.
+
     delete_all_watches(client)
 
 # Custom notification body with HTML, that is either sent as HTML or rendered to plaintext and sent
@@ -752,7 +782,6 @@ def test_check_html_notification_with_apprise_format_is_html(client, live_server
     text_content = text_part.get_content()
     assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
     assert 'fallback-body\r\n' in text_content  # The plaintext part
-    assert CUSTOM_LINEBREAK_PLACEHOLDER not in text_content
 
     # Second part should be text/html
     html_part = parts[1]
@@ -761,5 +790,4 @@ def test_check_html_notification_with_apprise_format_is_html(client, live_server
     assert 'some text<br>' in html_content  # We converted \n from the notification body
     assert 'fallback-body<br>' in html_content  # kept the original <br>
     assert '(added) So let\'s see what happens.<br>' in html_content  # the html part
-    assert CUSTOM_LINEBREAK_PLACEHOLDER not in html_content
     delete_all_watches(client)

changedetectionio/tests/test_notification.py

@@ -404,15 +404,15 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
 
 
 #2510
+#@todo run it again as text, html, htmlcolor
 def test_global_send_test_notification(client, live_server, measure_memory_usage, datastore_path):
 
-    
     set_original_response(datastore_path=datastore_path)
     if os.path.isfile(os.path.join(datastore_path, "notification.txt")):
         os.unlink(os.path.join(datastore_path, "notification.txt")) \
 
     # 1995 UTF-8 content should be encoded
-    test_body = 'change detection is cool 网站监测 内容更新了'
+    test_body = 'change detection is cool 网站监测 内容更新了 - {{diff_full}}'
 
     # otherwise other settings would have already existed from previous tests in this file
     res = client.post(
@@ -452,7 +452,14 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
 
     with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
         x = f.read()
-        assert test_body in x
+        assert 'change detection is cool 网站监测 内容更新了' in x
+        if 'html' in default_notification_format:
+            # this should come from default text when in global/system mode here changedetectionio/notification_service.py
+            assert 'title="Changed into">Example text:' in x
+        else:
+            assert 'title="Changed into">Example text:' not in x
+            assert 'span' not in x
+            assert 'Example text:' in x
 
     os.unlink(os.path.join(datastore_path, "notification.txt"))
 
@@ -509,6 +516,47 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert b"Error: You must have atleast one watch configured for 'test notification' to work" in res.data
 
 
+#2510
+def test_single_send_test_notification_on_watch(client, live_server, measure_memory_usage, datastore_path):
+
+    set_original_response(datastore_path=datastore_path)
+    if os.path.isfile(os.path.join(datastore_path, "notification.txt")):
+        os.unlink(os.path.join(datastore_path, "notification.txt")) \
+
+
+    test_url = url_for('test_endpoint', _external=True)
+    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
+    # 1995 UTF-8 content should be encoded
+    test_body = 'change detection is cool 网站监测 内容更新了 - {{diff_full}}'
+    ######### Test global/system settings
+    res = client.post(
+        url_for("ui.ui_notification.ajax_callback_send_notification_test")+f"/{uuid}",
+        data={"notification_urls": test_notification_url,
+              "notification_body": test_body,
+              "notification_format": default_notification_format,
+              "notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
+              },
+        follow_redirects=True
+    )
+
+    assert res.status_code != 400
+    assert res.status_code != 500
+
+    with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
+        x = f.read()
+        assert 'change detection is cool 网站监测 内容更新了' in x
+        if 'html' in default_notification_format:
+            # this should come from default text when in global/system mode here changedetectionio/notification_service.py
+            assert 'title="Changed into">Example text:' in x
+        else:
+            assert 'title="Changed into">Example text:' not in x
+            assert 'span' not in x
+            assert 'Example text:' in x
+
+    os.unlink(os.path.join(datastore_path, "notification.txt"))
 
 def _test_color_notifications(client, notification_body_token, datastore_path):
 

docs/api-spec.yaml

@@ -228,6 +228,11 @@ components:
                 maxLength: 5000
             required: [operation, selector, optional_value]
           description: Browser automation steps
+        processor:
+          type: string
+          enum: [restock_diff, text_json_diff]
+          default: text_json_diff
+          description: Optional processor mode to use for change detection. Defaults to `text_json_diff` if not specified.
 
     Watch:
       allOf:
@@ -428,7 +433,7 @@ paths:
       operationId: createWatch
       tags: [Watch Management]
       summary: Create a new watch
-      description: Create a single web page change monitor (watch). Requires at least 'url' to be set.
+      description: Create a single web page change monitor (watch). Requires at least 'url' to be set, Optionally use `"processor"` field to set the `restock_diff` mode or `text_json_diff` (default)
       x-code-samples:
         - lang: 'curl'
           source: |
@@ -446,7 +451,7 @@ paths:
           source: |
             import requests
             import json
-            
+
             headers = {
                 'x-api-key': 'YOUR_API_KEY',
                 'Content-Type': 'application/json'
@@ -458,7 +463,7 @@ paths:
                     'hours': 1
                 }
             }
-            response = requests.post('http://localhost:5000/api/v1/watch', 
+            response = requests.post('http://localhost:5000/api/v1/watch',
                                    headers=headers, json=data)
             print(response.text)
       requestBody: