mirror of
				https://github.com/dgtlmoon/changedetection.io.git
				synced 2025-10-30 14:17:40 +00:00 
			
		
		
		
	Compare commits
	
		
			17 Commits
		
	
	
		
			3540-html-
			...
			filter-not
		
	
	| Author | SHA1 | Date | |
|---|---|---|---|
|   | 69af4edb0f | ||
|   | 2d3a1bc53b | ||
|   | ce66ef1a9b | ||
|   | 8f580ac96b | ||
|   | a8cadc3d16 | ||
|   | c9290d73e0 | ||
|   | 2db5e906e9 | ||
|   | 0751bd371a | ||
| ![dependabot[bot]](/assets/img/avatar_default.png)  | 3ffa0805e9 | ||
| ![dependabot[bot]](/assets/img/avatar_default.png)  | 3335270692 | ||
| ![dependabot[bot]](/assets/img/avatar_default.png)  | a7573b10ec | ||
| ![dependabot[bot]](/assets/img/avatar_default.png)  | df945ad743 | ||
| ![dependabot[bot]](/assets/img/avatar_default.png)  | 4536e95205 | ||
|   | 1479d7bd46 | ||
|   | 9ba2094f75 | ||
|   | 8aa012ba8e | ||
|   | 8bc6b10db1 | 
							
								
								
									
										6
									
								
								.github/workflows/pypi-release.yml
									
									
									
									
										vendored
									
									
								
							
							
						
						
									
										6
									
								
								.github/workflows/pypi-release.yml
									
									
									
									
										vendored
									
									
								
							| @@ -21,7 +21,7 @@ jobs: | ||||
|     - name: Build a binary wheel and a source tarball | ||||
|       run: python3 -m build | ||||
|     - name: Store the distribution packages | ||||
|       uses: actions/upload-artifact@v4 | ||||
|       uses: actions/upload-artifact@v5 | ||||
|       with: | ||||
|         name: python-package-distributions | ||||
|         path: dist/ | ||||
| @@ -34,7 +34,7 @@ jobs: | ||||
|     - build | ||||
|     steps: | ||||
|     - name: Download all the dists | ||||
|       uses: actions/download-artifact@v5 | ||||
|       uses: actions/download-artifact@v6 | ||||
|       with: | ||||
|         name: python-package-distributions | ||||
|         path: dist/ | ||||
| @@ -93,7 +93,7 @@ jobs: | ||||
|  | ||||
|     steps: | ||||
|     - name: Download all the dists | ||||
|       uses: actions/download-artifact@v5 | ||||
|       uses: actions/download-artifact@v6 | ||||
|       with: | ||||
|         name: python-package-distributions | ||||
|         path: dist/ | ||||
|   | ||||
| @@ -282,7 +282,7 @@ jobs: | ||||
|  | ||||
|       - name: Store everything including test-datastore | ||||
|         if: always() | ||||
|         uses: actions/upload-artifact@v4 | ||||
|         uses: actions/upload-artifact@v5 | ||||
|         with: | ||||
|           name: test-cdio-basic-tests-output-py${{ env.PYTHON_VERSION }} | ||||
|           path: . | ||||
|   | ||||
| @@ -2,7 +2,7 @@ | ||||
|  | ||||
| # Read more https://github.com/dgtlmoon/changedetection.io/wiki | ||||
|  | ||||
| __version__ = '0.50.31' | ||||
| __version__ = '0.50.33' | ||||
|  | ||||
| from changedetectionio.strtobool import strtobool | ||||
| from json.decoder import JSONDecodeError | ||||
|   | ||||
| @@ -3,15 +3,30 @@ from changedetectionio.strtobool import strtobool | ||||
| from flask_restful import abort, Resource | ||||
| from flask import request | ||||
| import validators | ||||
| from functools import wraps | ||||
| from . import auth, validate_openapi_request | ||||
|  | ||||
|  | ||||
| def default_content_type(content_type='text/plain'): | ||||
|     """Decorator to set a default Content-Type header if none is provided.""" | ||||
|     def decorator(f): | ||||
|         @wraps(f) | ||||
|         def wrapper(*args, **kwargs): | ||||
|             if not request.content_type: | ||||
|                 # Set default content type in the request environment | ||||
|                 request.environ['CONTENT_TYPE'] = content_type | ||||
|             return f(*args, **kwargs) | ||||
|         return wrapper | ||||
|     return decorator | ||||
|  | ||||
|  | ||||
| class Import(Resource): | ||||
|     def __init__(self, **kwargs): | ||||
|         # datastore is a black box dependency | ||||
|         self.datastore = kwargs['datastore'] | ||||
|  | ||||
|     @auth.check_token | ||||
|     @default_content_type('text/plain') #3547 #3542 | ||||
|     @validate_openapi_request('importWatches') | ||||
|     def post(self): | ||||
|         """Import a list of watched URLs.""" | ||||
|   | ||||
| @@ -240,9 +240,7 @@ nav | ||||
|                     <p> | ||||
|                        {{ render_field(form.application.form.scheduler_timezone_default) }} | ||||
|                         <datalist id="timezones" style="display: none;"> | ||||
|                             {% for tz_name in available_timezones %} | ||||
|                                 <option value="{{ tz_name }}">{{ tz_name }}</option> | ||||
|                             {% endfor %} | ||||
|                             {%- for timezone in available_timezones -%}<option value="{{ timezone }}">{{ timezone }}</option>{%- endfor -%} | ||||
|                         </datalist> | ||||
|                     </p> | ||||
|                 </div> | ||||
|   | ||||
| @@ -76,14 +76,14 @@ def _handle_operations(op, uuids, datastore, worker_handler, update_q, queuedWat | ||||
|  | ||||
|     elif (op == 'notification-default'): | ||||
|         from changedetectionio.notification import ( | ||||
|             default_notification_format_for_watch | ||||
|             USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH | ||||
|         ) | ||||
|         for uuid in uuids: | ||||
|             if datastore.data['watching'].get(uuid): | ||||
|                 datastore.data['watching'][uuid]['notification_title'] = None | ||||
|                 datastore.data['watching'][uuid]['notification_body'] = None | ||||
|                 datastore.data['watching'][uuid]['notification_urls'] = [] | ||||
|                 datastore.data['watching'][uuid]['notification_format'] = default_notification_format_for_watch | ||||
|                 datastore.data['watching'][uuid]['notification_format'] = USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH | ||||
|         if emit_flash: | ||||
|             flash(f"{len(uuids)} watches set to use default notification settings") | ||||
|  | ||||
|   | ||||
| @@ -328,8 +328,9 @@ Math: {{ 1 + 1 }}") }} | ||||
|                         {{ render_checkbox_field(form.filter_text_replaced) }} | ||||
|                         {{ render_checkbox_field(form.filter_text_removed) }} | ||||
|                     <span class="pure-form-message-inline">Note: Depending on the length and similarity of the text on each line, the algorithm may consider an <strong>addition</strong> instead of <strong>replacement</strong> for example.</span><br> | ||||
|                     <span class="pure-form-message-inline"> So it's always better to select <strong>Added</strong>+<strong>Replaced</strong> when you're interested in new content.</span><br> | ||||
|                     <span class="pure-form-message-inline"> When content is merely moved in a list, it will also trigger an <strong>addition</strong>, consider enabling <code><strong>Only trigger when unique lines appear</strong></code></span> | ||||
|                     <span class="pure-form-message-inline">So it's always better to select <strong>Added</strong>+<strong>Replaced</strong> when you're interested in new content.</span><br> | ||||
|                     <span class="pure-form-message-inline">When content is merely moved in a list, it will also trigger an <strong>addition</strong>, consider enabling <code><strong>Only trigger when unique lines appear</strong></code>.</span><br> | ||||
|                     <span class="pure-form-message-inline">The full snapshot is still saved (this does not strip added/changed/removed lines), only limits the triggers of the change detection.</span> | ||||
|                 </fieldset> | ||||
|                 <fieldset class="pure-control-group"> | ||||
|                     {{ render_checkbox_field(form.check_unique_lines) }} | ||||
|   | ||||
| @@ -75,7 +75,6 @@ class Fetcher(): | ||||
|         self.screenshot = None | ||||
|         self.xpath_data = None | ||||
|         # Keep headers and status_code as they're small | ||||
|         logger.trace("Fetcher content cleared from memory") | ||||
|  | ||||
|     @abstractmethod | ||||
|     def get_error(self): | ||||
|   | ||||
| @@ -741,7 +741,6 @@ class quickWatchForm(Form): | ||||
|     edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"}) | ||||
|  | ||||
|  | ||||
|  | ||||
| # Common to a single watch and the global settings | ||||
| class commonSettingsForm(Form): | ||||
|     from . import processors | ||||
| @@ -754,7 +753,7 @@ class commonSettingsForm(Form): | ||||
|  | ||||
|     fetch_backend = RadioField(u'Fetch Method', choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()]) | ||||
|     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()]) | ||||
|     notification_format = SelectField('Notification format', choices=valid_notification_formats.keys()) | ||||
|     notification_format = SelectField('Notification format', choices=list(valid_notification_formats.items())) | ||||
|     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()]) | ||||
|     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()]) | ||||
|     processor = RadioField( label=u"Processor - What do you want to achieve?", choices=processors.available_processors(), default="text_json_diff") | ||||
|   | ||||
| @@ -2,7 +2,7 @@ import os | ||||
| import uuid | ||||
|  | ||||
| from changedetectionio import strtobool | ||||
| default_notification_format_for_watch = 'System default' | ||||
| USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH = 'System default' | ||||
| CONDITIONS_MATCH_LOGIC_DEFAULT = 'ALL' | ||||
|  | ||||
| class watch_base(dict): | ||||
| @@ -44,7 +44,7 @@ class watch_base(dict): | ||||
|             'method': 'GET', | ||||
|             'notification_alert_count': 0, | ||||
|             'notification_body': None, | ||||
|             'notification_format': default_notification_format_for_watch, | ||||
|             'notification_format': USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH, | ||||
|             'notification_muted': False, | ||||
|             'notification_screenshot': False,  # Include the latest screenshot if available and supported by the apprise URL | ||||
|             'notification_title': None, | ||||
|   | ||||
| @@ -1,17 +1,16 @@ | ||||
| from changedetectionio.model import default_notification_format_for_watch | ||||
| from changedetectionio.model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH | ||||
|  | ||||
| default_notification_format = 'HTML Color' | ||||
| default_notification_format = 'htmlcolor' | ||||
| default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n' | ||||
| default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}' | ||||
|  | ||||
| # The values (markdown etc) are from apprise NotifyFormat, | ||||
| # But to avoid importing the whole heavy module just use the same strings here. | ||||
| valid_notification_formats = { | ||||
|     'Plain Text': 'text', | ||||
|     'HTML': 'html', | ||||
|     'HTML Color': 'htmlcolor', | ||||
|     'Markdown to HTML': 'markdown', | ||||
|     'text': 'Plain Text', | ||||
|     'html': 'HTML', | ||||
|     'htmlcolor': 'HTML Color', | ||||
|     'markdown': 'Markdown to HTML', | ||||
|     # Used only for editing a watch (not for global) | ||||
|     default_notification_format_for_watch: default_notification_format_for_watch | ||||
|     USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH: USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH | ||||
| } | ||||
|  | ||||
|   | ||||
							
								
								
									
										42
									
								
								changedetectionio/notification/email_helpers.py
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										42
									
								
								changedetectionio/notification/email_helpers.py
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,42 @@ | ||||
| def as_monospaced_html_email(content: str, title: str) -> str: | ||||
|     """ | ||||
|     Wraps `content` in a minimal, email-safe HTML template | ||||
|     that forces monospace rendering across Gmail, Hotmail, Apple Mail, etc. | ||||
|  | ||||
|     Args: | ||||
|         content: The body text (plain text or HTML-like). | ||||
|         title: The title plaintext | ||||
|     Returns: | ||||
|         A complete HTML document string suitable for sending as an email body. | ||||
|     """ | ||||
|  | ||||
|     # All line feed types should be removed and then this function should only be fed <br>'s | ||||
|     # Then it works with our <pre> styling without double linefeeds | ||||
|     content = content.translate(str.maketrans('', '', '\r\n')) | ||||
|  | ||||
|     if title: | ||||
|         import html | ||||
|         title = html.escape(title) | ||||
|     else: | ||||
|         title = '' | ||||
|     # 2. Full email-safe HTML | ||||
|     html_email = f"""<!DOCTYPE html> | ||||
| <html lang="en"> | ||||
| <head> | ||||
|   <meta charset="UTF-8"> | ||||
|   <meta name="x-apple-disable-message-reformatting"> | ||||
|   <meta name="viewport" content="width=device-width, initial-scale=1.0"> | ||||
|   <!--[if mso]> | ||||
|     <style> | ||||
|       body, div, pre, td {{ font-family: "Courier New", Courier, monospace !important; }} | ||||
|     </style> | ||||
|   <![endif]--> | ||||
|   <title>{title}</title> | ||||
| </head> | ||||
| <body style="-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%;"> | ||||
|   <pre role="article" aria-roledescription="email" lang="en" | ||||
|        style="font-family: monospace, 'Courier New', Courier; font-size: 0.8em; | ||||
|               white-space: pre-wrap; word-break: break-word;">{content}</pre> | ||||
| </body> | ||||
| </html>""" | ||||
|     return html_email | ||||
| @@ -6,6 +6,7 @@ from loguru import logger | ||||
| from urllib.parse import urlparse | ||||
| from .apprise_plugin.assets import apprise_asset, APPRISE_AVATAR_URL | ||||
| from .apprise_plugin.custom_handlers import SUPPORTED_HTTP_METHODS | ||||
| from .email_helpers import as_monospaced_html_email | ||||
| from ..diff import HTML_REMOVED_STYLE, REMOVED_PLACEMARKER_OPEN, REMOVED_PLACEMARKER_CLOSED, ADDED_PLACEMARKER_OPEN, HTML_ADDED_STYLE, \ | ||||
|     ADDED_PLACEMARKER_CLOSED, CHANGED_INTO_PLACEMARKER_OPEN, CHANGED_INTO_PLACEMARKER_CLOSED, CHANGED_PLACEMARKER_OPEN, \ | ||||
|     CHANGED_PLACEMARKER_CLOSED, HTML_CHANGED_STYLE, HTML_CHANGED_INTO_STYLE | ||||
| @@ -62,13 +63,13 @@ def notification_format_align_with_apprise(n_format : str): | ||||
|     :return: | ||||
|     """ | ||||
|  | ||||
|     if n_format.lower().startswith('html'): | ||||
|     if n_format.startswith('html'): | ||||
|         # Apprise only knows 'html' not 'htmlcolor' etc, which shouldnt matter here | ||||
|         n_format = NotifyFormat.HTML.value | ||||
|     elif n_format.lower().startswith('markdown'): | ||||
|     elif n_format.startswith('markdown'): | ||||
|         # probably the same but just to be safe | ||||
|         n_format = NotifyFormat.MARKDOWN.value | ||||
|     elif n_format.lower().startswith('text'): | ||||
|     elif n_format.startswith('text'): | ||||
|         # probably the same but just to be safe | ||||
|         n_format = NotifyFormat.TEXT.value | ||||
|     else: | ||||
| @@ -76,6 +77,55 @@ def notification_format_align_with_apprise(n_format : str): | ||||
|  | ||||
|     return n_format | ||||
|  | ||||
| def apply_discord_markdown_to_body(n_body): | ||||
|     """ | ||||
|     Discord does not support <del> but it supports non-standard ~~strikethrough~~ | ||||
|     :param n_body: | ||||
|     :return: | ||||
|     """ | ||||
|     import re | ||||
|     # Define the mapping between your placeholders and markdown markers | ||||
|     replacements = [ | ||||
|         (REMOVED_PLACEMARKER_OPEN, '~~', REMOVED_PLACEMARKER_CLOSED, '~~'), | ||||
|         (ADDED_PLACEMARKER_OPEN, '**', ADDED_PLACEMARKER_CLOSED, '**'), | ||||
|         (CHANGED_PLACEMARKER_OPEN, '~~', CHANGED_PLACEMARKER_CLOSED, '~~'), | ||||
|         (CHANGED_INTO_PLACEMARKER_OPEN, '**', CHANGED_INTO_PLACEMARKER_CLOSED, '**'), | ||||
|     ] | ||||
|     # So that the markdown gets added without any whitespace following it which would break it | ||||
|     for open_tag, open_md, close_tag, close_md in replacements: | ||||
|         # Regex: match opening tag, optional whitespace, capture the content, optional whitespace, then closing tag | ||||
|         pattern = re.compile( | ||||
|             re.escape(open_tag) + r'(\s*)(.*?)?(\s*)' + re.escape(close_tag), | ||||
|             flags=re.DOTALL | ||||
|         ) | ||||
|         n_body = pattern.sub(lambda m: f"{m.group(1)}{open_md}{m.group(2)}{close_md}{m.group(3)}", n_body) | ||||
|     return n_body | ||||
|  | ||||
| def apply_standard_markdown_to_body(n_body): | ||||
|     """ | ||||
|     Apprise does not support ~~strikethrough~~ but it will convert <del> to HTML strikethrough. | ||||
|     :param n_body: | ||||
|     :return: | ||||
|     """ | ||||
|     import re | ||||
|     # Define the mapping between your placeholders and markdown markers | ||||
|     replacements = [ | ||||
|         (REMOVED_PLACEMARKER_OPEN, '<del>', REMOVED_PLACEMARKER_CLOSED, '</del>'), | ||||
|         (ADDED_PLACEMARKER_OPEN, '**', ADDED_PLACEMARKER_CLOSED, '**'), | ||||
|         (CHANGED_PLACEMARKER_OPEN, '<del>', CHANGED_PLACEMARKER_CLOSED, '</del>'), | ||||
|         (CHANGED_INTO_PLACEMARKER_OPEN, '**', CHANGED_INTO_PLACEMARKER_CLOSED, '**'), | ||||
|     ] | ||||
|  | ||||
|     # So that the markdown gets added without any whitespace following it which would break it | ||||
|     for open_tag, open_md, close_tag, close_md in replacements: | ||||
|         # Regex: match opening tag, optional whitespace, capture the content, optional whitespace, then closing tag | ||||
|         pattern = re.compile( | ||||
|             re.escape(open_tag) + r'(\s*)(.*?)?(\s*)' + re.escape(close_tag), | ||||
|             flags=re.DOTALL | ||||
|         ) | ||||
|         n_body = pattern.sub(lambda m: f"{m.group(1)}{open_md}{m.group(2)}{close_md}{m.group(3)}", n_body) | ||||
|     return n_body | ||||
|  | ||||
|  | ||||
| def apply_service_tweaks(url, n_body, n_title, requested_output_format): | ||||
|  | ||||
| @@ -106,7 +156,7 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format): | ||||
|         n_body = n_body.replace('<br>', '\n') | ||||
|         n_body = n_body.replace('</br>', '\n') | ||||
|         n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\n') | ||||
|          | ||||
|  | ||||
|         # Use strikethrough for removed content, bold for added content | ||||
|         n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '<s>') | ||||
|         n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '</s>') | ||||
| @@ -140,15 +190,7 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format): | ||||
|         if requested_output_format == 'html': | ||||
|             # No diff placeholders, use Discord markdown for any other formatting | ||||
|             # Use Discord markdown: strikethrough for removed, bold for added | ||||
|             n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '~~') | ||||
|             n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '~~') | ||||
|             n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '**') | ||||
|             n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '**') | ||||
|             # Handle changed/replaced lines (old → new) | ||||
|             n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, '~~') | ||||
|             n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, '~~') | ||||
|             n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, '**') | ||||
|             n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, '**') | ||||
|             n_body = apply_discord_markdown_to_body(n_body=n_body) | ||||
|  | ||||
|             # Apply 2000 char limit for plain content | ||||
|             payload_max_size = 1700 | ||||
| @@ -180,6 +222,9 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format): | ||||
|         n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ') | ||||
|         n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'') | ||||
|         n_body = n_body.replace('\n', f'{CUSTOM_LINEBREAK_PLACEHOLDER}\n') | ||||
|     elif requested_output_format == 'markdown': | ||||
|         # Markdown to HTML - Apprise will convert this to HTML | ||||
|         n_body = apply_standard_markdown_to_body(n_body=n_body) | ||||
|  | ||||
|     else: #plaintext etc default | ||||
|         n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ') | ||||
| @@ -196,7 +241,7 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format): | ||||
|  | ||||
| def process_notification(n_object: NotificationContextData, datastore): | ||||
|     from changedetectionio.jinja2_custom import render as jinja_render | ||||
|     from . import default_notification_format_for_watch, default_notification_format, valid_notification_formats | ||||
|     from . import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH, default_notification_format, valid_notification_formats | ||||
|     # be sure its registered | ||||
|     from .apprise_plugin.custom_handlers import apprise_http_custom_handler | ||||
|     # Register custom Discord plugin | ||||
| @@ -212,18 +257,17 @@ def process_notification(n_object: NotificationContextData, datastore): | ||||
|     # Insert variables into the notification content | ||||
|     notification_parameters = create_notification_parameters(n_object, datastore) | ||||
|  | ||||
|     requested_output_format = valid_notification_formats.get( | ||||
|         n_object.get('notification_format', default_notification_format), | ||||
|         valid_notification_formats[default_notification_format], | ||||
|     ) | ||||
|     requested_output_format = n_object.get('notification_format', default_notification_format) | ||||
|     logger.debug(f"Requested notification output format: '{requested_output_format}'") | ||||
|  | ||||
|     # If we arrived with 'System default' then look it up | ||||
|     if requested_output_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch: | ||||
|     if requested_output_format == USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH: | ||||
|         # Initially text or whatever | ||||
|         requested_output_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format]).lower() | ||||
|         requested_output_format = datastore.data['settings']['application'].get('notification_format', default_notification_format) | ||||
|  | ||||
|     requested_output_format_original = requested_output_format | ||||
|  | ||||
|     # Now clean it up so it fits perfectly with apprise | ||||
|     requested_output_format = notification_format_align_with_apprise(n_format=requested_output_format) | ||||
|  | ||||
|     logger.trace(f"Complete notification body including Jinja and placeholders calculated in  {time.time() - now:.2f}s") | ||||
| @@ -300,17 +344,20 @@ def process_notification(n_object: NotificationContextData, datastore): | ||||
|                     apprise_input_format = NotifyFormat.TEXT.value | ||||
|  | ||||
|                 elif requested_output_format == NotifyFormat.MARKDOWN.value: | ||||
|                     # This actually means we request "Markdown to HTML", we want HTML output | ||||
|                     # Convert markdown to HTML ourselves since not all plugins do this | ||||
|                     from apprise.conversion import markdown_to_html | ||||
|                     # Make sure there are paragraph breaks around horizontal rules | ||||
|                     n_body = n_body.replace('---', '\n\n---\n\n') | ||||
|                     n_body = markdown_to_html(n_body) | ||||
|                     url = f"{url}{prefix_add_to_url}format={NotifyFormat.HTML.value}" | ||||
|                     requested_output_format = NotifyFormat.HTML.value | ||||
|                     apprise_input_format = NotifyFormat.MARKDOWN.value | ||||
|  | ||||
|                     apprise_input_format = NotifyFormat.HTML.value  # Changed from MARKDOWN to HTML | ||||
|  | ||||
|                 # Could have arrived at any stage, so we dont end up running .escape on it | ||||
|                 if 'html' in requested_output_format: | ||||
|                     n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '<br>\n') | ||||
|                     n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '<br>\r\n') | ||||
|                 else: | ||||
|                     # Markup, text types etc | ||||
|                     # texty types | ||||
|                     n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\r\n') | ||||
|  | ||||
|             sent_objs.append({'title': n_title, | ||||
| @@ -318,6 +365,12 @@ def process_notification(n_object: NotificationContextData, datastore): | ||||
|                               'url': url}) | ||||
|             apobj.add(url) | ||||
|  | ||||
|             # Since the output is always based on the plaintext of the 'diff' engine, wrap it nicely. | ||||
|             # It should always be similar to the 'history' part of the UI. | ||||
|             if url.startswith('mail') and 'html' in requested_output_format: | ||||
|                 if not '<pre' in n_body and not '<body' in n_body: # No custom HTML-ish body was setup already | ||||
|                     n_body = as_monospaced_html_email(content=n_body, title=n_title) | ||||
|  | ||||
|         apobj.notify( | ||||
|             title=n_title, | ||||
|             body=n_body, | ||||
|   | ||||
| @@ -9,7 +9,8 @@ for both sync and async workers | ||||
| from loguru import logger | ||||
| import time | ||||
|  | ||||
| from changedetectionio.notification import default_notification_format | ||||
| from changedetectionio.model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH | ||||
| from changedetectionio.notification import default_notification_format, valid_notification_formats | ||||
|  | ||||
| # This gets modified on notification time (handler.py) depending on the required notification output | ||||
| CUSTOM_LINEBREAK_PLACEHOLDER='@BR@' | ||||
| @@ -48,15 +49,28 @@ class NotificationContextData(dict): | ||||
|         if kwargs: | ||||
|             self.update(kwargs) | ||||
|  | ||||
|         n_format = self.get('notification_format') | ||||
|         if n_format and not valid_notification_formats.get(n_format): | ||||
|             raise ValueError(f'Invalid notification format: "{n_format}"') | ||||
|  | ||||
|     def set_random_for_validation(self): | ||||
|         import random, string | ||||
|         """Randomly fills all dict keys with random strings (for validation/testing).""" | ||||
|         """Randomly fills all dict keys with random strings (for validation/testing).  | ||||
|         So we can test the output in the notification body | ||||
|         """ | ||||
|         for key in self.keys(): | ||||
|             if key in ['uuid', 'time', 'watch_uuid']: | ||||
|                 continue | ||||
|             rand_str = 'RANDOM-PLACEHOLDER-'+''.join(random.choices(string.ascii_letters + string.digits, k=12)) | ||||
|             self[key] = rand_str | ||||
|  | ||||
|     def __setitem__(self, key, value): | ||||
|         if key == 'notification_format' and isinstance(value, str) and not value.startswith('RANDOM-PLACEHOLDER-'): | ||||
|             if not valid_notification_formats.get(value): | ||||
|                 raise ValueError(f'Invalid notification format: "{value}"') | ||||
|  | ||||
|         super().__setitem__(key, value) | ||||
|  | ||||
| class NotificationService: | ||||
|     """ | ||||
|     Standalone notification service that handles all notification functionality | ||||
| @@ -72,7 +86,7 @@ class NotificationService: | ||||
|         Queue a notification for a watch with full diff rendering and template variables | ||||
|         """ | ||||
|         from changedetectionio import diff | ||||
|         from changedetectionio.notification import default_notification_format_for_watch | ||||
|         from changedetectionio.notification import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH | ||||
|  | ||||
|         if not isinstance(n_object, NotificationContextData): | ||||
|             raise TypeError(f"Expected NotificationContextData, got {type(n_object)}") | ||||
| @@ -94,7 +108,7 @@ class NotificationService: | ||||
|             snapshot_contents = "No snapshot/history available, the watch should fetch atleast once." | ||||
|  | ||||
|         # If we ended up here with "System default" | ||||
|         if n_object.get('notification_format') == default_notification_format_for_watch: | ||||
|         if n_object.get('notification_format') == USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH: | ||||
|             n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format') | ||||
|  | ||||
|  | ||||
| @@ -141,7 +155,7 @@ class NotificationService: | ||||
|         Individual watch settings > Tag settings > Global settings | ||||
|         """ | ||||
|         from changedetectionio.notification import ( | ||||
|             default_notification_format_for_watch, | ||||
|             USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH, | ||||
|             default_notification_body, | ||||
|             default_notification_title | ||||
|         ) | ||||
| @@ -149,7 +163,7 @@ class NotificationService: | ||||
|         # Would be better if this was some kind of Object where Watch can reference the parent datastore etc | ||||
|         v = watch.get(var_name) | ||||
|         if v and not watch.get('notification_muted'): | ||||
|             if var_name == 'notification_format' and v == default_notification_format_for_watch: | ||||
|             if var_name == 'notification_format' and v == USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH: | ||||
|                 return self.datastore.data['settings']['application'].get('notification_format') | ||||
|  | ||||
|             return v | ||||
| @@ -166,7 +180,7 @@ class NotificationService: | ||||
|  | ||||
|         # Otherwise could be defaults | ||||
|         if var_name == 'notification_format': | ||||
|             return default_notification_format_for_watch | ||||
|             return USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH | ||||
|         if var_name == 'notification_body': | ||||
|             return default_notification_body | ||||
|         if var_name == 'notification_title': | ||||
| @@ -221,7 +235,6 @@ class NotificationService: | ||||
|         if not watch: | ||||
|             return | ||||
|  | ||||
|         n_format = self.datastore.data['settings']['application'].get('notification_format', default_notification_format) | ||||
|         filter_list = ", ".join(watch['include_filters']) | ||||
|         # @todo - This could be a markdown template on the disk, apprise will convert the markdown to HTML+Plaintext parts in the email, and then 'markup_text_links_to_html_links' is not needed | ||||
|         body = f"""Hello, | ||||
| @@ -238,9 +251,9 @@ Thanks - Your omniscient changedetection.io installation. | ||||
|         n_object = NotificationContextData({ | ||||
|             'notification_title': 'Changedetection.io - Alert - CSS/xPath filter was not present in the page', | ||||
|             'notification_body': body, | ||||
|             'notification_format': n_format, | ||||
|             'markup_text_links_to_html_links': n_format.lower().startswith('html') | ||||
|             'notification_format': self._check_cascading_vars('notification_format', watch), | ||||
|         }) | ||||
|         n_object['markup_text_links_to_html_links'] = n_object.get('notification_format').startswith('html') | ||||
|  | ||||
|         if len(watch['notification_urls']): | ||||
|             n_object['notification_urls'] = watch['notification_urls'] | ||||
| @@ -268,7 +281,7 @@ Thanks - Your omniscient changedetection.io installation. | ||||
|         if not watch: | ||||
|             return | ||||
|         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts') | ||||
|         n_format = self.datastore.data['settings']['application'].get('notification_format', default_notification_format).lower() | ||||
|  | ||||
|         step = step_n + 1 | ||||
|         # @todo - This could be a markdown template on the disk, apprise will convert the markdown to HTML+Plaintext parts in the email, and then 'markup_text_links_to_html_links' is not needed | ||||
|  | ||||
| @@ -287,9 +300,9 @@ Thanks - Your omniscient changedetection.io installation. | ||||
|         n_object = NotificationContextData({ | ||||
|             'notification_title': f"Changedetection.io - Alert - Browser step at position {step} could not be run", | ||||
|             'notification_body': body, | ||||
|             'notification_format': n_format, | ||||
|             'markup_text_links_to_html_links': n_format.lower().startswith('html') | ||||
|             'notification_format': self._check_cascading_vars('notification_format', watch), | ||||
|         }) | ||||
|         n_object['markup_text_links_to_html_links'] = n_object.get('notification_format').startswith('html') | ||||
|  | ||||
|         if len(watch['notification_urls']): | ||||
|             n_object['notification_urls'] = watch['notification_urls'] | ||||
|   | ||||
| @@ -459,15 +459,27 @@ class perform_site_check(difference_detection_processor): | ||||
|         # Save text before ignore filters (for diff calculation) | ||||
|         text_content_before_ignored_filter = stripped_text | ||||
|  | ||||
|         # Save full content before diff filtering for consistent MD5 calculation | ||||
|         full_content_for_md5 = None | ||||
|  | ||||
|         # === DIFF FILTERING === | ||||
|         # If user wants specific diff types (added/removed/replaced only) | ||||
|         if watch.has_special_diff_filter_options_set() and len(watch.history.keys()): | ||||
|             stripped_text = self._apply_diff_filtering(watch, stripped_text, text_content_before_ignored_filter) | ||||
|             if stripped_text is None: | ||||
|                 # No differences found, but content exists | ||||
|             # Save full content BEFORE applying diff filtering | ||||
|             # This ensures MD5 is always calculated from full content, not the filtered diff | ||||
|             full_content_for_md5 = stripped_text | ||||
|  | ||||
|             filtered_diff = self._apply_diff_filtering(watch, stripped_text, text_content_before_ignored_filter) | ||||
|             if filtered_diff is None: | ||||
|                 # No matching differences found (e.g., only removed lines when user wants added/replaced) | ||||
|                 # Calculate MD5 of full content and return early | ||||
|                 c = ChecksumCalculator.calculate(text_content_before_ignored_filter, ignore_whitespace=True) | ||||
|                 return False, {'previous_md5': c}, text_content_before_ignored_filter.encode('utf-8') | ||||
|  | ||||
|             # Has matching changes - use filtered diff for trigger_text evaluation and display, | ||||
|             # but full_content_for_md5 will be used later for MD5 calculation | ||||
|             stripped_text = filtered_diff | ||||
|  | ||||
|         # === EMPTY PAGE CHECK === | ||||
|         empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False) | ||||
|         if not stream_content_type.is_json and not empty_pages_are_a_change and len(stripped_text.strip()) == 0: | ||||
| @@ -495,17 +507,23 @@ class perform_site_check(difference_detection_processor): | ||||
|             stripped_text = transformer.sort_alphabetically(stripped_text) | ||||
|  | ||||
|         # === CHECKSUM CALCULATION === | ||||
|         text_for_checksuming = stripped_text | ||||
|         # When diff filtering is active, use full content for MD5, not the filtered diff | ||||
|         # This ensures consistent MD5 calculation regardless of what changes occurred | ||||
|         if full_content_for_md5 is not None: | ||||
|             text_for_checksuming = full_content_for_md5 | ||||
|         else: | ||||
|             text_for_checksuming = stripped_text | ||||
|  | ||||
|         # Apply ignore_text for checksum calculation | ||||
|         if filter_config.ignore_text: | ||||
|             text_for_checksuming = html_tools.strip_ignore_text(stripped_text, filter_config.ignore_text) | ||||
|             text_for_checksuming = html_tools.strip_ignore_text(text_for_checksuming, filter_config.ignore_text) | ||||
|  | ||||
|             # Optionally remove ignored lines from output | ||||
|             # Note: Only apply to stripped_text if we're not using full_content_for_md5 | ||||
|             strip_ignored_lines = watch.get('strip_ignored_lines') | ||||
|             if strip_ignored_lines is None: | ||||
|                 strip_ignored_lines = self.datastore.data['settings']['application'].get('strip_ignored_lines') | ||||
|             if strip_ignored_lines: | ||||
|             if strip_ignored_lines and full_content_for_md5 is None: | ||||
|                 stripped_text = text_for_checksuming | ||||
|  | ||||
|         # Calculate checksum | ||||
| @@ -571,7 +589,12 @@ class perform_site_check(difference_detection_processor): | ||||
|         if 'text_for_checksuming' in locals() and text_for_checksuming is not stripped_text: | ||||
|             del text_for_checksuming | ||||
|  | ||||
|         return changed_detected, update_obj, stripped_text | ||||
|         # When diff filtering is active, return full content for history snapshots | ||||
|         # stripped_text contains only the filtered diff, which would create confusing/broken snapshots | ||||
|         if full_content_for_md5 is not None: | ||||
|             return changed_detected, update_obj, full_content_for_md5 | ||||
|         else: | ||||
|             return changed_detected, update_obj, stripped_text | ||||
|  | ||||
|     def _apply_diff_filtering(self, watch, stripped_text, text_before_filter): | ||||
|         """Apply user's diff filtering preferences (show only added/removed/replaced lines).""" | ||||
|   | ||||
| @@ -5,7 +5,7 @@ from flask import ( | ||||
| ) | ||||
|  | ||||
| from .html_tools import TRANSLATE_WHITESPACE_TABLE | ||||
| from . model import App, Watch | ||||
| from .model import App, Watch, USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH | ||||
| from copy import deepcopy, copy | ||||
| from os import path, unlink | ||||
| from threading import Lock | ||||
| @@ -987,10 +987,35 @@ class ChangeDetectionStore: | ||||
|             self.data['settings']['application']['ui']['use_page_title_in_list'] = self.data['settings']['application'].get('extract_title_as_title') | ||||
|  | ||||
|     def update_21(self): | ||||
|         self.data['settings']['application']['scheduler_timezone_default'] = self.data['settings']['application'].get('timezone') | ||||
|         del self.data['settings']['application']['timezone'] | ||||
|         if self.data['settings']['application'].get('timezone'): | ||||
|             self.data['settings']['application']['scheduler_timezone_default'] = self.data['settings']['application'].get('timezone') | ||||
|             del self.data['settings']['application']['timezone'] | ||||
|  | ||||
|  | ||||
|     # Some notification formats got the wrong name type | ||||
|     def update_22(self): | ||||
|         from .notification import valid_notification_formats | ||||
|  | ||||
|         sys_n_format = self.data['settings']['application'].get('notification_format') | ||||
|         key_exists_as_value = next((k for k, v in valid_notification_formats.items() if v == sys_n_format), None) | ||||
|         if key_exists_as_value: # key of "Plain text" | ||||
|             logger.success(f"['settings']['application']['notification_format'] '{sys_n_format}' -> '{key_exists_as_value}'") | ||||
|             self.data['settings']['application']['notification_format'] = key_exists_as_value | ||||
|  | ||||
|         for uuid, watch in self.data['watching'].items(): | ||||
|             n_format = self.data['watching'][uuid].get('notification_format') | ||||
|             key_exists_as_value = next((k for k, v in valid_notification_formats.items() if v == n_format), None) | ||||
|             if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of "Plain text" | ||||
|                 logger.success(f"['watching'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'") | ||||
|                 self.data['watching'][uuid]['notification_format'] = key_exists_as_value # should be 'text' or whatever | ||||
|  | ||||
|         for uuid, tag in self.data['settings']['application']['tags'].items(): | ||||
|             n_format = self.data['settings']['application']['tags'][uuid].get('notification_format') | ||||
|             key_exists_as_value = next((k for k, v in valid_notification_formats.items() if v == n_format), None) | ||||
|             if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of "Plain text" | ||||
|                 logger.success(f"['settings']['application']['tags'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'") | ||||
|                 self.data['settings']['application']['tags'][uuid]['notification_format'] = key_exists_as_value # should be 'text' or whatever | ||||
|  | ||||
|     def add_notification_url(self, notification_url): | ||||
|          | ||||
|         logger.debug(f">>> Adding new notification_url - '{notification_url}'") | ||||
|   | ||||
| @@ -266,9 +266,7 @@ | ||||
|             <li id="timezone-info"> | ||||
|                 {{ render_field(form.time_schedule_limit.timezone, placeholder=timezone_default_config) }} <span id="local-time-in-tz"></span> | ||||
|                 <datalist id="timezones" style="display: none;"> | ||||
|                     {% for timezone in available_timezones %} | ||||
|                         <option value="{{ timezone }}">{{ timezone }}</option> | ||||
|                     {% endfor %} | ||||
|                     {%- for timezone in available_timezones -%}<option value="{{ timezone }}">{{ timezone }}</option>{%- endfor -%} | ||||
|                 </datalist> | ||||
|             </li> | ||||
|         </ul> | ||||
|   | ||||
| @@ -53,7 +53,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas | ||||
|         data={"application-notification_urls": notification_url, | ||||
|               "application-notification_title": "fallback-title " + default_notification_title, | ||||
|               "application-notification_body": "some text\nfallback-body<br> " + default_notification_body, | ||||
|               "application-notification_format": 'HTML', | ||||
|               "application-notification_format": 'html', | ||||
|               "requests-time_between_check-minutes": 180, | ||||
|               'application-fetch_backend': "html_requests"}, | ||||
|         follow_redirects=True | ||||
| @@ -122,7 +122,7 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory | ||||
|         data={"application-notification_urls": notification_url, | ||||
|               "application-notification_title": "fallback-title " + default_notification_title, | ||||
|               "application-notification_body": "some text\n" + default_notification_body, | ||||
|               "application-notification_format": 'Plain Text', | ||||
|               "application-notification_format": 'text', | ||||
|               "requests-time_between_check-minutes": 180, | ||||
|               'application-fetch_backend': "html_requests"}, | ||||
|         follow_redirects=True | ||||
| @@ -174,7 +174,7 @@ def test_check_notification_html_color_format(client, live_server, measure_memor | ||||
|         data={"application-notification_urls": notification_url, | ||||
|               "application-notification_title": "fallback-title " + default_notification_title, | ||||
|               "application-notification_body": f"some text\n{default_notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}", | ||||
|               "application-notification_format": 'HTML Color', | ||||
|               "application-notification_format": 'htmlcolor', | ||||
|               "requests-time_between_check-minutes": 180, | ||||
|               'application-fetch_backend': "html_requests"}, | ||||
|         follow_redirects=True | ||||
| @@ -245,7 +245,7 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_ | ||||
|         data={"application-notification_urls": notification_url, | ||||
|               "application-notification_title": "fallback-title " + default_notification_title, | ||||
|               "application-notification_body": "*header*\n\nsome text\n" + default_notification_body, | ||||
|               "application-notification_format": 'Markdown to HTML', | ||||
|               "application-notification_format": 'markdown', | ||||
|               "requests-time_between_check-minutes": 180, | ||||
|               'application-fetch_backend': "html_requests"}, | ||||
|         follow_redirects=True | ||||
| @@ -290,7 +290,8 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_ | ||||
|     text_part = parts[0] | ||||
|     assert text_part.get_content_type() == 'text/plain' | ||||
|     text_content = text_part.get_content() | ||||
|     assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part | ||||
|     # We wont see anything in the "FALLBACK" text but that's OK (no added/strikethrough etc) | ||||
|     assert 'So let\'s see what happens.\r\n' in text_content  # The plaintext part | ||||
|  | ||||
|  | ||||
|     # Second part should be text/html and roughly converted from markdown to HTML | ||||
| @@ -298,7 +299,7 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_ | ||||
|     assert html_part.get_content_type() == 'text/html' | ||||
|     html_content = html_part.get_content() | ||||
|     assert '<p><em>header</em></p>' in html_content | ||||
|     assert '(added) So let\'s see what happens.<br' in html_content | ||||
|     assert '<strong>So let\'s see what happens.</strong><br>' in html_content # Additions are <strong> in markdown | ||||
|     delete_all_watches(client) | ||||
|  | ||||
| # Custom notification body with HTML, that is either sent as HTML or rendered to plaintext and sent | ||||
| @@ -328,7 +329,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv | ||||
|         data={"application-notification_urls": notification_url, | ||||
|               "application-notification_title": "fallback-title " + default_notification_title, | ||||
|               "application-notification_body": notification_body, | ||||
|               "application-notification_format": 'Plain Text', | ||||
|               "application-notification_format": 'text', | ||||
|               "requests-time_between_check-minutes": 180, | ||||
|               'application-fetch_backend': "html_requests"}, | ||||
|         follow_redirects=True | ||||
| @@ -378,7 +379,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv | ||||
|         url_for("ui.ui_edit.edit_page", uuid="first"), | ||||
|         data={ | ||||
|             "url": test_url, | ||||
|             "notification_format": 'HTML', | ||||
|             "notification_format": 'html', | ||||
|             'fetch_backend': "html_requests", | ||||
|             "time_between_check_use_default": "y"}, | ||||
|         follow_redirects=True | ||||
| @@ -437,7 +438,7 @@ def test_check_plaintext_document_plaintext_notification_smtp(client, live_serve | ||||
|         data={"application-notification_urls": notification_url, | ||||
|               "application-notification_title": "fallback-title " + default_notification_title, | ||||
|               "application-notification_body": f"{notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}", | ||||
|               "application-notification_format": 'Plain Text', | ||||
|               "application-notification_format": 'text', | ||||
|               "requests-time_between_check-minutes": 180, | ||||
|               'application-fetch_backend': "html_requests"}, | ||||
|         follow_redirects=True | ||||
| @@ -470,7 +471,7 @@ def test_check_plaintext_document_plaintext_notification_smtp(client, live_serve | ||||
|     assert '(added)' in body | ||||
|     assert '<br' not in body | ||||
|     assert '<' not in body | ||||
|  | ||||
|     assert '<pre' not in body | ||||
|     delete_all_watches(client) | ||||
|  | ||||
| def test_check_plaintext_document_html_notifications(client, live_server, measure_memory_usage): | ||||
| @@ -489,7 +490,7 @@ def test_check_plaintext_document_html_notifications(client, live_server, measur | ||||
|         data={"application-notification_urls": notification_url, | ||||
|               "application-notification_title": "fallback-title " + default_notification_title, | ||||
|               "application-notification_body": f"{notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}", | ||||
|               "application-notification_format": 'HTML', | ||||
|               "application-notification_format": 'html', | ||||
|               "requests-time_between_check-minutes": 180, | ||||
|               'application-fetch_backend': "html_requests"}, | ||||
|         follow_redirects=True | ||||
| @@ -541,13 +542,13 @@ def test_check_plaintext_document_html_notifications(client, live_server, measur | ||||
|     assert 'talk about <title>' in html_content | ||||
|     # Should be the HTML, but not HTML Color | ||||
|     assert 'background-color' not in html_content | ||||
|     assert '<br>\r\n(added) And let's talk about <title> tags<br>' in html_content | ||||
|     assert '<br>(added) And let's talk about <title> tags<br>' in html_content | ||||
|     assert '<br' not in html_content | ||||
|     assert '<pre role="article"' in html_content # Should have got wrapped nicely in email_helpers.py | ||||
|  | ||||
|     # And now for the whitespace retention | ||||
|     assert '    Some nice plain text' in html_content | ||||
|     assert '(added) And let' in html_content # just to show a single whitespace didnt get touched | ||||
|  | ||||
|     delete_all_watches(client) | ||||
|  | ||||
|  | ||||
| @@ -567,7 +568,7 @@ def test_check_plaintext_document_html_color_notifications(client, live_server, | ||||
|         data={"application-notification_urls": notification_url, | ||||
|               "application-notification_title": "fallback-title " + default_notification_title, | ||||
|               "application-notification_body": f"{notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}", | ||||
|               "application-notification_format": 'HTML Color', | ||||
|               "application-notification_format": 'htmlcolor', | ||||
|               "requests-time_between_check-minutes": 180, | ||||
|               'application-fetch_backend': "html_requests"}, | ||||
|         follow_redirects=True | ||||
| @@ -620,7 +621,7 @@ def test_check_plaintext_document_html_color_notifications(client, live_server, | ||||
|     assert '(added) And let' not in html_content | ||||
|     assert '<br' not in html_content | ||||
|     assert '<br>' in html_content | ||||
|  | ||||
|     assert '<pre role="article"' in html_content # Should have got wrapped nicely in email_helpers.py | ||||
|     delete_all_watches(client) | ||||
|  | ||||
| def test_check_html_document_plaintext_notification(client, live_server, measure_memory_usage): | ||||
| @@ -639,7 +640,7 @@ def test_check_html_document_plaintext_notification(client, live_server, measure | ||||
|         data={"application-notification_urls": notification_url, | ||||
|               "application-notification_title": "fallback-title " + default_notification_title, | ||||
|               "application-notification_body": f"{notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}", | ||||
|               "application-notification_format": 'Plain Text', | ||||
|               "application-notification_format": 'text', | ||||
|               "requests-time_between_check-minutes": 180, | ||||
|               'application-fetch_backend': "html_requests"}, | ||||
|         follow_redirects=True | ||||
|   | ||||
| @@ -124,7 +124,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa | ||||
|               "application-notification_body": 'triggered text was -{{triggered_text}}- ### 网站监测 内容更新了 ####', | ||||
|               # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation | ||||
|               "application-notification_urls": test_notification_url, | ||||
|               "application-notification_format": 'Plain Text', | ||||
|               "application-notification_format": 'text', | ||||
|               "application-minutes_between_check": 180, | ||||
|               "application-fetch_backend": "html_requests" | ||||
|               }, | ||||
| @@ -183,3 +183,104 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa | ||||
|         assert '网站监测 内容更新了'.encode('utf-8') in response | ||||
|  | ||||
|     delete_all_watches(client) | ||||
|  | ||||
|  | ||||
| def test_consistent_md5_with_diff_filtering(client, live_server, measure_memory_usage): | ||||
|     """ | ||||
|     Test that MD5 checksums are calculated consistently when diff filtering is active. | ||||
|  | ||||
|     This test ensures that after a change is detected with diff filtering enabled, | ||||
|     subsequent checks with identical content don't trigger false positives. | ||||
|  | ||||
|     Bug: Previously, MD5 was calculated from the filtered diff (partial content) | ||||
|     when changes were found, but from full content when no changes were found. | ||||
|     This caused false positives on the next check with identical content. | ||||
|  | ||||
|     Fix: Always calculate MD5 from full content, regardless of diff filtering. | ||||
|     """ | ||||
|  | ||||
|     delete_all_watches(client) | ||||
|     time.sleep(1) | ||||
|  | ||||
|     # Setup initial content | ||||
|     set_original() | ||||
|     test_url = url_for('test_endpoint', _external=True) | ||||
|     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url) | ||||
|  | ||||
|     # Configure: Only track ADDED and REPLACED lines, ignore REMOVED lines | ||||
|     res = client.post( | ||||
|         url_for("ui.ui_edit.edit_page", uuid="first"), | ||||
|         data={ | ||||
|             "url": test_url, | ||||
|             'processor': 'text_json_diff', | ||||
|             'fetch_backend': "html_requests", | ||||
|             'filter_text_added': 'y',       # Track added lines | ||||
|             'filter_text_replaced': 'y',    # Track replaced lines | ||||
|             'filter_text_removed': '',      # Don't track removed lines | ||||
|             "time_between_check_use_default": "y" | ||||
|         }, | ||||
|         follow_redirects=True | ||||
|     ) | ||||
|     assert b"Updated watch." in res.data | ||||
|  | ||||
|     # CHECK 1: Initial baseline | ||||
|     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) | ||||
|     wait_for_all_checks(client) | ||||
|     res = client.get(url_for("watchlist.index")) | ||||
|     assert b'has-unread-changes' not in res.data  # First check, no change | ||||
|  | ||||
|     # Mark as viewed to start fresh | ||||
|     client.get(url_for("ui.mark_all_viewed"), follow_redirects=True) | ||||
|  | ||||
|     # CHECK 2: Remove a line (should NOT trigger - removed lines are filtered out) | ||||
|     set_original(excluding='Something irrelevant') | ||||
|     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) | ||||
|     wait_for_all_checks(client) | ||||
|     res = client.get(url_for("watchlist.index")) | ||||
|     assert b'has-unread-changes' not in res.data  # No change (removed line filtered) | ||||
|  | ||||
|     # CHECK 3: Add a line (should trigger - added lines are tracked) | ||||
|     set_original(excluding='Something irrelevant', add_line='<p>New exciting feature!</p>') | ||||
|     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) | ||||
|     wait_for_all_checks(client) | ||||
|     res = client.get(url_for("watchlist.index")) | ||||
|     assert b'has-unread-changes' in res.data  # Change detected (added line) | ||||
|  | ||||
|     # Mark as viewed | ||||
|     client.get(url_for("ui.mark_all_viewed"), follow_redirects=True) | ||||
|  | ||||
|     # CHECK 4: Same content as CHECK 3 (THE CRITICAL TEST - should NOT trigger) | ||||
|     # This is where the bug would manifest: false positive change detection | ||||
|     # because previous MD5 was from filtered diff, current MD5 is from full content | ||||
|     set_original(excluding='Something irrelevant', add_line='<p>New exciting feature!</p>') | ||||
|     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) | ||||
|     wait_for_all_checks(client) | ||||
|     res = client.get(url_for("watchlist.index")) | ||||
|  | ||||
|     # CRITICAL ASSERTION: Should NOT detect change (content is identical to CHECK 3) | ||||
|     assert b'has-unread-changes' not in res.data, \ | ||||
|         "False positive! Content identical to previous check but change was detected. " \ | ||||
|         "MD5 calculation is inconsistent with diff filtering." | ||||
|  | ||||
|     # CHECK 5: Verify system still detects real changes (replace a line) | ||||
|     # Change "Some initial text" to "Some modified text" | ||||
|     modified_content = """<html> | ||||
|      <body> | ||||
|      <p>Some modified text</p> | ||||
|      <p>So let's see what happens.</p> | ||||
|      <p>and a new line!</p> | ||||
|      <p>The golden line</p> | ||||
|      <p>New exciting feature!</p> | ||||
|      <p>A BREAK TO MAKE THE TOP LINE STAY AS "REMOVED" OR IT WILL GET COUNTED AS "CHANGED INTO"</p> | ||||
|      </body> | ||||
|      </html> | ||||
|     """ | ||||
|     with open("test-datastore/endpoint-content.txt", "w") as f: | ||||
|         f.write(modified_content) | ||||
|  | ||||
|     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) | ||||
|     wait_for_all_checks(client) | ||||
|     res = client.get(url_for("watchlist.index")) | ||||
|     assert b'has-unread-changes' in res.data  # Change detected (replaced line) | ||||
|  | ||||
|     delete_all_watches(client) | ||||
|   | ||||
| @@ -394,7 +394,8 @@ def test_api_import(client, live_server, measure_memory_usage): | ||||
|     res = client.post( | ||||
|         url_for("import") + "?tag=import-test", | ||||
|         data='https://website1.com\r\nhttps://website2.com', | ||||
|         headers={'x-api-key': api_key, 'content-type': 'text/plain'}, | ||||
|         # We removed  'content-type': 'text/plain', the Import API should assume this if none is set #3547 #3542 | ||||
|         headers={'x-api-key': api_key}, | ||||
|         follow_redirects=True | ||||
|     ) | ||||
|  | ||||
|   | ||||
| @@ -86,7 +86,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se | ||||
|                                                    "Diff Full: {{diff_full}}\n" | ||||
|                                                    "Diff as Patch: {{diff_patch}}\n" | ||||
|                                                    ":-)", | ||||
|                               "notification_format": 'Plain Text'} | ||||
|                               "notification_format": 'text'} | ||||
|  | ||||
|     notification_form_data.update({ | ||||
|         "url": test_url, | ||||
|   | ||||
| @@ -63,7 +63,7 @@ def run_filter_test(client, live_server, content_filter, app_notification_format | ||||
|                                        "Diff Full: {{diff_full}}\n" | ||||
|                                        "Diff as Patch: {{diff_patch}}\n" | ||||
|                                        ":-)", | ||||
|                   "notification_format": 'Plain Text', | ||||
|                   "notification_format": 'text', | ||||
|                   "fetch_backend": "html_requests", | ||||
|                   "filter_failure_notification_send": 'y', | ||||
|                   "time_between_check_use_default": "y", | ||||
| @@ -175,13 +175,13 @@ def run_filter_test(client, live_server, content_filter, app_notification_format | ||||
|  | ||||
| def test_check_include_filters_failure_notification(client, live_server, measure_memory_usage): | ||||
|     #   #  live_server_setup(live_server) # Setup on conftest per function | ||||
|     run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('HTML Color')) | ||||
|     run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('htmlcolor')) | ||||
|     # Check markup send conversion didnt affect plaintext preference | ||||
|     run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('Plain Text')) | ||||
|     run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('text')) | ||||
|  | ||||
| def test_check_xpath_filter_failure_notification(client, live_server, measure_memory_usage): | ||||
|     #   #  live_server_setup(live_server) # Setup on conftest per function | ||||
|     run_filter_test(client=client, live_server=live_server, content_filter='//*[@id="nope-doesnt-exist"]', app_notification_format=valid_notification_formats.get('HTML Color')) | ||||
|     run_filter_test(client=client, live_server=live_server, content_filter='//*[@id="nope-doesnt-exist"]', app_notification_format=valid_notification_formats.get('htmlcolor')) | ||||
|  | ||||
| # Test that notification is never sent | ||||
|  | ||||
|   | ||||
| @@ -195,7 +195,7 @@ def test_group_tag_notification(client, live_server, measure_memory_usage): | ||||
|                                                    "Diff as Patch: {{diff_patch}}\n" | ||||
|                                                    ":-)", | ||||
|                               "notification_screenshot": True, | ||||
|                               "notification_format": 'Plain Text', | ||||
|                               "notification_format": 'text', | ||||
|                               "title": "test-tag"} | ||||
|  | ||||
|     res = client.post( | ||||
|   | ||||
| @@ -13,10 +13,10 @@ import base64 | ||||
| from changedetectionio.notification import ( | ||||
|     default_notification_body, | ||||
|     default_notification_format, | ||||
|     default_notification_title, | ||||
|     valid_notification_formats, | ||||
|     default_notification_title, valid_notification_formats | ||||
| ) | ||||
| from ..diff import HTML_CHANGED_STYLE | ||||
| from ..model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH | ||||
|  | ||||
|  | ||||
| # Hard to just add more live server URLs when one test is already running (I think) | ||||
| @@ -47,6 +47,14 @@ def test_check_notification(client, live_server, measure_memory_usage): | ||||
|  | ||||
|     assert b"Settings updated." in res.data | ||||
|  | ||||
|     res = client.get(url_for("settings.settings_page")) | ||||
|     for k,v in valid_notification_formats.items(): | ||||
|         if k == USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH: | ||||
|             continue | ||||
|         assert f'value="{k}"'.encode() in res.data # Should be by key NOT value | ||||
|         assert f'value="{v}"'.encode() not in res.data # Should be by key NOT value | ||||
|  | ||||
|  | ||||
|     # When test mode is in BASE_URL env mode, we should see this already configured | ||||
|     env_base_url = os.getenv('BASE_URL', '').strip() | ||||
|     if len(env_base_url): | ||||
| @@ -101,7 +109,7 @@ def test_check_notification(client, live_server, measure_memory_usage): | ||||
|                                                    "Diff as Patch: {{diff_patch}}\n" | ||||
|                                                    ":-)", | ||||
|                               "notification_screenshot": True, | ||||
|                               "notification_format": 'Plain Text'} | ||||
|                               "notification_format": 'text'} | ||||
|  | ||||
|     notification_form_data.update({ | ||||
|         "url": test_url, | ||||
| @@ -267,7 +275,7 @@ def test_notification_validation(client, live_server, measure_memory_usage): | ||||
| #        data={"notification_urls": 'json://localhost/foobar', | ||||
| #              "notification_title": "", | ||||
| #              "notification_body": "", | ||||
| #              "notification_format": 'Plain Text', | ||||
| #              "notification_format": 'text', | ||||
| #              "url": test_url, | ||||
| #              "tag": "my tag", | ||||
| #              "title": "my title", | ||||
| @@ -521,7 +529,7 @@ def _test_color_notifications(client, notification_body_token): | ||||
|             "application-fetch_backend": "html_requests", | ||||
|             "application-minutes_between_check": 180, | ||||
|             "application-notification_body": notification_body_token, | ||||
|             "application-notification_format": "HTML Color", | ||||
|             "application-notification_format": "htmlcolor", | ||||
|             "application-notification_urls": test_notification_url, | ||||
|             "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}", | ||||
|         }, | ||||
|   | ||||
| @@ -30,7 +30,7 @@ def test_check_notification_error_handling(client, live_server, measure_memory_u | ||||
|         data={"notification_urls": f"{broken_notification_url}\r\n{working_notification_url}", | ||||
|               "notification_title": "xxx", | ||||
|               "notification_body": "xxxxx", | ||||
|               "notification_format": 'Plain Text', | ||||
|               "notification_format": 'text', | ||||
|               "url": test_url, | ||||
|               "tags": "", | ||||
|               "title": "", | ||||
|   | ||||
| @@ -2,6 +2,9 @@ | ||||
|  | ||||
| import sys | ||||
| import os | ||||
|  | ||||
| from changedetectionio.model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH | ||||
|  | ||||
| sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../..')) | ||||
|  | ||||
| from changedetectionio.widgets import TernaryNoneBooleanField | ||||
| @@ -93,7 +96,7 @@ def test_custom_text(): | ||||
|     print(f"Does NOT contain 'System default': {'System default' not in boolean_html}") | ||||
|     print(f"Does NOT contain 'Default': {'Default' not in boolean_html}") | ||||
|     assert 'Enabled' in boolean_html and 'Disabled' in boolean_html | ||||
|     assert 'System default' not in boolean_html and 'Default' not in boolean_html | ||||
|     assert USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH not in boolean_html and 'Default' not in boolean_html | ||||
|      | ||||
|     # Test FontAwesome field | ||||
|     print("\n--- FontAwesome Icons Field ---") | ||||
|   | ||||
| @@ -28,7 +28,7 @@ info: | ||||
|      | ||||
|     For example: `x-api-key: YOUR_API_KEY` | ||||
|      | ||||
|   version: 0.1.1 | ||||
|   version: 0.1.2 | ||||
|   contact: | ||||
|     name: ChangeDetection.io | ||||
|     url: https://github.com/dgtlmoon/changedetection.io | ||||
| @@ -143,7 +143,7 @@ components: | ||||
|         paused: | ||||
|           type: boolean | ||||
|           description: Whether the web page change monitor (watch) is paused | ||||
|         muted: | ||||
|         notification_muted: | ||||
|           type: boolean | ||||
|           description: Whether notifications are muted | ||||
|         method: | ||||
| @@ -207,7 +207,7 @@ components: | ||||
|           maxLength: 5000 | ||||
|         notification_format: | ||||
|           type: string | ||||
|           enum: [Text, HTML, Markdown] | ||||
|           enum: ['text', 'html', 'htmlcolor', 'markdown', 'System default'] | ||||
|           description: Format for notifications | ||||
|         track_ldjson_price_data: | ||||
|           type: boolean | ||||
| @@ -406,7 +406,7 @@ paths: | ||||
|                   page_title: "The HTML <title> from the page" | ||||
|                   tags: ["550e8400-e29b-41d4-a716-446655440000"] | ||||
|                   paused: false | ||||
|                   muted: false | ||||
|                   notification_muted: false | ||||
|                   method: "GET" | ||||
|                   fetch_backend: "html_requests" | ||||
|                   last_checked: 1640995200 | ||||
| @@ -419,7 +419,7 @@ paths: | ||||
|                   page_title: "The HTML <title> from the page" | ||||
|                   tags: ["330e8400-e29b-41d4-a716-446655440001"] | ||||
|                   paused: false | ||||
|                   muted: true | ||||
|                   notification_muted: true | ||||
|                   method: "GET" | ||||
|                   fetch_backend: "html_webdriver" | ||||
|                   last_checked: 1640998800 | ||||
| @@ -1224,7 +1224,7 @@ paths: | ||||
|                     title: "Example Website Monitor" | ||||
|                     tags: ["550e8400-e29b-41d4-a716-446655440000"] | ||||
|                     paused: false | ||||
|                     muted: false | ||||
|                     notification_muted: false | ||||
|  | ||||
|   /import: | ||||
|     post: | ||||
|   | ||||
| @@ -1,5 +1,5 @@ | ||||
| # eventlet>=0.38.0  # Removed - replaced with threading mode for better Python 3.12+ compatibility | ||||
| feedgen~=0.9 | ||||
| feedgen~=1.0 | ||||
| feedparser~=6.0  # For parsing RSS/Atom feeds | ||||
| flask-compress | ||||
| # 0.6.3 included compatibility fix for werkzeug 3.x (2.x had deprecation of url handlers) | ||||
| @@ -12,7 +12,7 @@ janus # Thread-safe async/sync queue bridge | ||||
| flask_wtf~=1.2 | ||||
| flask~=3.1 | ||||
| flask-socketio~=5.5.1 | ||||
| python-socketio~=5.13.0 | ||||
| python-socketio~=5.14.2 | ||||
| python-engineio~=4.12.3 | ||||
| inscriptis~=2.2 | ||||
| pytz | ||||
| @@ -22,7 +22,7 @@ validators~=0.35 | ||||
|  | ||||
| # Set these versions together to avoid a RequestsDependencyWarning | ||||
| # >= 2.26 also adds Brotli support if brotli is installed | ||||
| brotli~=1.0 | ||||
| brotli~=1.1 | ||||
| requests[socks] | ||||
| requests-file | ||||
|  | ||||
| @@ -30,7 +30,7 @@ requests-file | ||||
| # If specific version needed for security, use urllib3>=1.26.19,<3.0 | ||||
| chardet>2.3.0 | ||||
|  | ||||
| wtforms~=3.0 | ||||
| wtforms~=3.2 | ||||
| jsonpath-ng~=1.5.3 | ||||
|  | ||||
| # dnspython - Used by paho-mqtt for MQTT broker resolution   | ||||
|   | ||||
		Reference in New Issue
	
	Block a user