diff --git a/changedetectionio/blueprint/rss/__init__.py b/changedetectionio/blueprint/rss/__init__.py index adecd339..fe20cc81 100644 --- a/changedetectionio/blueprint/rss/__init__.py +++ b/changedetectionio/blueprint/rss/__init__.py @@ -3,6 +3,7 @@ from loguru import logger from changedetectionio.model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH from changedetectionio.notification import valid_notification_formats + RSS_CONTENT_FORMAT_DEFAULT = 'text' # Some stuff not related @@ -15,3 +16,12 @@ if RSS_FORMAT_TYPES.get(USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH): if not RSS_FORMAT_TYPES.get(RSS_CONTENT_FORMAT_DEFAULT): logger.critical(f"RSS_CONTENT_FORMAT_DEFAULT not in the acceptable list {RSS_CONTENT_FORMAT_DEFAULT}") + +RSS_TEMPLATE_TYPE_OPTIONS = {'system_default': 'System default', 'notification_body': 'Notification body'} + +# @note: We use
 because nearly all RSS readers render only HTML (Thunderbird for example cant do just plaintext)
+RSS_TEMPLATE_PLAINTEXT_DEFAULT = "
{{watch_label}} had a change.\n\n{{diff}}\n
" + +# @todo add some [edit]/[history]/[goto] etc links +# @todo need {{watch_edit_link}} + delete + history link token +RSS_TEMPLATE_HTML_DEFAULT = "\n

{{watch_label}}

\n

{{diff}}

\n\n" diff --git a/changedetectionio/blueprint/rss/_util.py b/changedetectionio/blueprint/rss/_util.py index 8aa56dd2..c96f8b41 100644 --- a/changedetectionio/blueprint/rss/_util.py +++ b/changedetectionio/blueprint/rss/_util.py @@ -2,9 +2,11 @@ Utility functions for RSS feed generation. """ -from changedetectionio.jinja2_custom import render as jinja_render -from changedetectionio.notification.handler import apply_service_tweaks +from changedetectionio.notification.handler import process_notification +from changedetectionio.notification_service import NotificationContextData, _check_cascading_vars from loguru import logger +import datetime +import pytz import re @@ -39,59 +41,116 @@ def clean_entry_content(content): return cleaned -def generate_watch_guid(watch): +def generate_watch_guid(watch, timestamp): """ Generate a unique GUID for a watch RSS entry. - """ - return f"{watch['uuid']}/{watch.last_changed}" - - -def generate_watch_diff_content(watch, dates, rss_content_format, datastore, date_index_from=-2, date_index_to=-1): - """ - Generate HTML diff content for a watch given its history dates. - Returns tuple of (content, watch_label). Args: watch: The watch object - dates: List of history snapshot dates - rss_content_format: Format for RSS content (html or text) - datastore: The ChangeDetectionStore instance - date_index_from: Index of the "from" date in the dates list (default: -2) - date_index_to: Index of the "to" date in the dates list (default: -1) + timestamp: The timestamp of the specific change this entry represents + """ + return f"{watch['uuid']}/{timestamp}" + + +def validate_rss_token(datastore, request): + """ + Validate the RSS access token from the request. Returns: - Tuple of (content, watch_label) - the rendered HTML content and watch label + tuple: (is_valid, error_response) where error_response is None if valid """ - from changedetectionio import diff + app_rss_token = datastore.data['settings']['application'].get('rss_access_token') + rss_url_token = request.args.get('token') - # Same logic as watch-overview.html - if datastore.data['settings']['application']['ui'].get('use_page_title_in_list') or watch.get('use_page_title_in_list'): - watch_label = watch.label + if rss_url_token != app_rss_token: + return False, ("Access denied, bad token", 403) + + return True, None + + +def get_rss_template(datastore, watch, rss_content_format, default_html, default_plaintext): + """Get the appropriate template for RSS content.""" + if datastore.data['settings']['application'].get('rss_template_type') == 'notification_body': + return _check_cascading_vars(datastore=datastore, var_name='notification_body', watch=watch) + + override = datastore.data['settings']['application'].get('rss_template_override') + if override and override.strip(): + return override + elif 'text' in rss_content_format: + return default_plaintext else: - watch_label = watch.get('url') + return default_html - try: - html_diff = diff.render_diff( - previous_version_file_contents=watch.get_history_snapshot(timestamp=dates[date_index_from]), - newest_version_file_contents=watch.get_history_snapshot(timestamp=dates[date_index_to]), - include_equal=False - ) - requested_output_format = datastore.data['settings']['application'].get('rss_content_format') - url, html_diff, n_title = apply_service_tweaks(url='', n_body=html_diff, n_title=None, requested_output_format=requested_output_format) +def get_watch_label(datastore, watch): + """Get the label for a watch based on settings.""" + if datastore.data['settings']['application']['ui'].get('use_page_title_in_list') or watch.get('use_page_title_in_list'): + return watch.label + else: + return watch.get('url') - except FileNotFoundError as e: - html_diff = f"History snapshot file for watch {watch.get('uuid')}@{watch.last_changed} - '{watch.get('title')} not found." - # @note: We use
 because nearly all RSS readers render only HTML (Thunderbird for example cant do just plaintext)
-    rss_template = "
{{watch_label}} had a change.\n\n{{html_diff}}\n
" - if 'html' in rss_content_format: - rss_template = "\n

{{watch_label}}

\n

{{html_diff}}

\n\n" +def add_watch_categories(fe, watch, datastore): + """Add category tags to a feed entry based on watch tags.""" + for tag_uuid in watch.get('tags', []): + tag = datastore.data['settings']['application'].get('tags', {}).get(tag_uuid) + if tag and tag.get('title'): + fe.category(term=tag.get('title')) - content = jinja_render(template_str=rss_template, watch_label=watch_label, html_diff=html_diff, watch_url=watch.link) - # Out of range chars could also break feedgen +def build_notification_context(watch, timestamp_from, timestamp_to, watch_label, + n_body_template, rss_content_format): + """Build the notification context object.""" + return NotificationContextData(initial_data={ + 'notification_urls': ['null://just-sending-a-null-test-for-the-render-in-RSS'], + 'notification_body': n_body_template, + 'timestamp_to': timestamp_to, + 'timestamp_from': timestamp_from, + 'watch_label': watch_label, + 'notification_format': rss_content_format + }) + + +def render_notification(n_object, notification_service, watch, datastore, + date_index_from=None, date_index_to=None): + """Process and render the notification content.""" + kwargs = {'n_object': n_object, 'watch': watch} + + if date_index_from is not None and date_index_to is not None: + kwargs['date_index_from'] = date_index_from + kwargs['date_index_to'] = date_index_to + + n_object = notification_service.queue_notification_for_watch(**kwargs) + n_object['watch_mime_type'] = None + + res = process_notification(n_object=n_object, datastore=datastore) + return res[0] + + +def populate_feed_entry(fe, watch, content, guid, timestamp, link=None, title_suffix=None): + """Populate a feed entry with content and metadata.""" + watch_label = watch.get('url') # Already determined by caller + + # Set link + if link: + fe.link(link=link) + + # Set title + if title_suffix: + fe.title(title=f"{watch_label} - {title_suffix}") + else: + fe.title(title=watch_label) + + # Clean and set content if scan_invalid_chars_in_rss(content): content = clean_entry_content(content) + fe.content(content=content, type='CDATA') + + # Set GUID + fe.guid(guid, permalink=False) + + # Set pubDate using the timestamp of this specific change + dt = datetime.datetime.fromtimestamp(int(timestamp)) + dt = dt.replace(tzinfo=pytz.UTC) + fe.pubDate(dt) - return content, watch_label diff --git a/changedetectionio/blueprint/rss/main_feed.py b/changedetectionio/blueprint/rss/main_feed.py index fec9d9f1..6c24e66d 100644 --- a/changedetectionio/blueprint/rss/main_feed.py +++ b/changedetectionio/blueprint/rss/main_feed.py @@ -1,11 +1,5 @@ from flask import make_response, request, url_for, redirect -from feedgen.feed import FeedGenerator -from loguru import logger -import datetime -import pytz -import time -from ._util import generate_watch_guid, generate_watch_diff_content def construct_main_feed_routes(rss_blueprint, datastore): @@ -27,14 +21,24 @@ def construct_main_feed_routes(rss_blueprint, datastore): # from changedetectionio.auth_decorator import login_optionally_required @rss_blueprint.route("", methods=['GET']) def feed(): - now = time.time() - # Always requires token set - app_rss_token = datastore.data['settings']['application'].get('rss_access_token') - rss_url_token = request.args.get('token') - rss_content_format = datastore.data['settings']['application'].get('rss_content_format') + from feedgen.feed import FeedGenerator + from loguru import logger + import time - if rss_url_token != app_rss_token: - return "Access denied, bad token", 403 + from . import RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT + from ._util import (validate_rss_token, generate_watch_guid, get_rss_template, + get_watch_label, build_notification_context, render_notification, + populate_feed_entry, add_watch_categories) + from ...notification_service import NotificationService + + now = time.time() + + # Validate token + is_valid, error = validate_rss_token(datastore, request) + if not is_valid: + return error + + rss_content_format = datastore.data['settings']['application'].get('rss_content_format') limit_tag = request.args.get('tag', '').lower().strip() # Be sure limit_tag is a uuid @@ -52,7 +56,6 @@ def construct_main_feed_routes(rss_blueprint, datastore): continue if limit_tag and not limit_tag in watch['tags']: continue - watch['uuid'] = uuid sorted_watches.append(watch) sorted_watches.sort(key=lambda x: x.last_changed, reverse=False) @@ -61,6 +64,7 @@ def construct_main_feed_routes(rss_blueprint, datastore): fg.title('changedetection.io') fg.description('Feed description') fg.link(href='https://changedetection.io') + notification_service = NotificationService(datastore=datastore, notification_q=False) for watch in sorted_watches: @@ -72,36 +76,28 @@ def construct_main_feed_routes(rss_blueprint, datastore): if not watch.viewed: # Re #239 - GUID needs to be individual for each event # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228) - guid = generate_watch_guid(watch) - fe = fg.add_entry() - - # Include a link to the diff page, they will have to login here to see if password protection is enabled. - # Description is the page you watch, link takes you to the diff JS UI page - # Dict val base_url will get overriden with the env var if it is set. - ext_base_url = datastore.data['settings']['application'].get('active_base_url') - # @todo fix - - # Because we are called via whatever web server, flask should figure out the right path ( + watch_label = get_watch_label(datastore, watch) + timestamp_to = dates[-1] + timestamp_from = dates[-2] + guid = generate_watch_guid(watch, timestamp_to) + # Because we are called via whatever web server, flask should figure out the right path diff_link = {'href': url_for('ui.ui_views.diff_history_page', uuid=watch['uuid'], _external=True)} - fe.link(link=diff_link) + # Get template and build notification context + n_body_template = get_rss_template(datastore, watch, rss_content_format, + RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT) - content, watch_label = generate_watch_diff_content(watch, dates, rss_content_format, datastore) + n_object = build_notification_context(watch, timestamp_from, timestamp_to, + watch_label, n_body_template, rss_content_format) - fe.title(title=watch_label) - fe.content(content=content, type='CDATA') - fe.guid(guid, permalink=False) - dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key)) - dt = dt.replace(tzinfo=pytz.UTC) - fe.pubDate(dt) + # Render notification + res = render_notification(n_object, notification_service, watch, datastore) - # Add categories based on watch tags - for tag_uuid in watch.get('tags', []): - tag = datastore.data['settings']['application'].get('tags', {}).get(tag_uuid) - if tag: - tag_title = tag.get('title', '') - if tag_title: - fe.category(term=tag_title) + # Create and populate feed entry + fe = fg.add_entry() + populate_feed_entry(fe, watch, res['body'], guid, timestamp_to, link=diff_link) + fe.title(title=watch_label) # Override title to not include suffix + add_watch_categories(fe, watch, datastore) response = make_response(fg.rss_str()) response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8') diff --git a/changedetectionio/blueprint/rss/single_watch.py b/changedetectionio/blueprint/rss/single_watch.py index d47a2abb..3cfdb190 100644 --- a/changedetectionio/blueprint/rss/single_watch.py +++ b/changedetectionio/blueprint/rss/single_watch.py @@ -1,10 +1,3 @@ -from flask import make_response, request, url_for -from feedgen.feed import FeedGenerator -import datetime -import pytz -import locale - -from ._util import generate_watch_guid, generate_watch_diff_content def construct_single_watch_routes(rss_blueprint, datastore): @@ -18,18 +11,31 @@ def construct_single_watch_routes(rss_blueprint, datastore): @rss_blueprint.route("/watch/", methods=['GET']) def rss_single_watch(uuid): + import time + + from flask import make_response, request + from feedgen.feed import FeedGenerator + from loguru import logger + + from . import RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT + from ._util import (validate_rss_token, get_rss_template, get_watch_label, + build_notification_context, render_notification, + populate_feed_entry, add_watch_categories) + from ...notification_service import NotificationService + """ Display the most recent changes for a single watch as RSS feed. Returns RSS XML with multiple entries showing diffs between consecutive snapshots. The number of entries is controlled by the rss_diff_length setting. """ - # Always requires token set - app_rss_token = datastore.data['settings']['application'].get('rss_access_token') - rss_url_token = request.args.get('token') - rss_content_format = datastore.data['settings']['application'].get('rss_content_format') + now = time.time() - if rss_url_token != app_rss_token: - return "Access denied, bad token", 403 + # Validate token + is_valid, error = validate_rss_token(datastore, request) + if not is_valid: + return error + + rss_content_format = datastore.data['settings']['application'].get('rss_content_format') # Get the watch by UUID watch = datastore.data['watching'].get(uuid) @@ -57,8 +63,9 @@ def construct_single_watch_routes(rss_blueprint, datastore): # Set title: use "label (url)" if label differs from url, otherwise just url watch_url = watch.get('url', '') - watch_label = watch.label - if watch_label and watch_label != watch_url: + watch_label = get_watch_label(datastore, watch) + + if watch_label != watch_url: feed_title = f'changedetection.io - {watch_label} ({watch_url})' else: feed_title = f'changedetection.io - {watch_url}' @@ -70,6 +77,8 @@ def construct_single_watch_routes(rss_blueprint, datastore): # Loop through history and create RSS entries for each diff # Add entries in reverse order because feedgen reverses them # This way, the newest change appears first in the final RSS + + notification_service = NotificationService(datastore=datastore, notification_q=False) for i in range(num_diffs - 1, -1, -1): # Calculate indices for this diff (working backwards from newest) # i=0: compare dates[-2] to dates[-1] (most recent change) @@ -77,83 +86,30 @@ def construct_single_watch_routes(rss_blueprint, datastore): # etc. date_index_to = -(i + 1) date_index_from = -(i + 2) + timestamp_to = dates[date_index_to] + timestamp_from = dates[date_index_from] - try: - # Generate the diff content for this pair of snapshots - timestamp_to = dates[date_index_to] - timestamp_from = dates[date_index_from] + # Get template and build notification context + n_body_template = get_rss_template(datastore, watch, rss_content_format, + RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT) - content, watch_label = generate_watch_diff_content( - watch, dates, rss_content_format, datastore, - date_index_from=date_index_from, - date_index_to=date_index_to - ) + n_object = build_notification_context(watch, timestamp_from, timestamp_to, + watch_label, n_body_template, rss_content_format) - # Generate edit watch link and add to content - edit_watch_url = url_for('ui.ui_edit.edit_page', - uuid=watch['uuid'], - _external=True) + # Render notification with date indices + res = render_notification(n_object, notification_service, watch, datastore, + date_index_from, date_index_to) - # Add edit watch links at top and bottom of content - if 'html' in rss_content_format: - edit_link_html = f'

[edit watch]

' - # Insert after and before - content = content.replace('', f'\n{edit_link_html}', 1) - content = content.replace('', f'{edit_link_html}\n', 1) - else: - # For plain text format, add plain text links in separate
 blocks
-                    edit_link_top = f'
[edit watch] {edit_watch_url}
\n' - edit_link_bottom = f'\n
[edit watch] {edit_watch_url}
' - content = edit_link_top + content + edit_link_bottom - - # Create a unique GUID for this specific diff - guid = f"{watch['uuid']}/{timestamp_to}" - - fe = fg.add_entry() - - # Include a link to the diff page with specific versions - diff_link = {'href': url_for('ui.ui_views.diff_history_page', - uuid=watch['uuid'], - from_version=timestamp_from, - to_version=timestamp_to, - _external=True)} - fe.link(link=diff_link) - - # Format the date using locale-aware formatting with timezone - dt = datetime.datetime.fromtimestamp(int(timestamp_to)) - dt = dt.replace(tzinfo=pytz.UTC) - - # Get local timezone-aware datetime - local_tz = datetime.datetime.now().astimezone().tzinfo - local_dt = dt.astimezone(local_tz) - - # Format date with timezone - using strftime for locale awareness - try: - formatted_date = local_dt.strftime('%Y-%m-%d %H:%M:%S %Z') - except: - # Fallback if locale issues - formatted_date = local_dt.isoformat() - - # Use formatted date in title instead of "Change 1, 2, 3" - fe.title(title=f"{watch_label} - Change @ {formatted_date}") - fe.content(content=content, type='CDATA') - fe.guid(guid, permalink=False) - - # Use the timestamp of the "to" snapshot for pubDate - fe.pubDate(dt) - - # Add categories based on watch tags - for tag_uuid in watch.get('tags', []): - tag = datastore.data['settings']['application'].get('tags', {}).get(tag_uuid) - if tag: - tag_title = tag.get('title', '') - if tag_title: - fe.category(term=tag_title) - - except (IndexError, FileNotFoundError) as e: - # Skip this diff if we can't generate it - continue + # Create and populate feed entry + guid = f"{watch['uuid']}/{timestamp_to}" + fe = fg.add_entry() + title_suffix = f"Change @ {res['original_context']['change_datetime']}" + populate_feed_entry(fe, watch, res.get('body', ''), guid, timestamp_to, + link={'href': watch.get('url')}, title_suffix=title_suffix) + add_watch_categories(fe, watch, datastore) response = make_response(fg.rss_str()) response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8') + logger.debug(f"RSS Single watch built in {time.time()-now:.2f}s") + return response diff --git a/changedetectionio/blueprint/rss/tag.py b/changedetectionio/blueprint/rss/tag.py index 86aeb290..c6daf77b 100644 --- a/changedetectionio/blueprint/rss/tag.py +++ b/changedetectionio/blueprint/rss/tag.py @@ -1,12 +1,3 @@ -from flask import make_response, request, url_for -from feedgen.feed import FeedGenerator -import datetime -import pytz - - -from ._util import generate_watch_guid, generate_watch_diff_content - - def construct_tag_routes(rss_blueprint, datastore): """ Construct RSS feed routes for tags. @@ -18,17 +9,26 @@ def construct_tag_routes(rss_blueprint, datastore): @rss_blueprint.route("/tag/", methods=['GET']) def rss_tag_feed(tag_uuid): + + from flask import make_response, request, url_for + from feedgen.feed import FeedGenerator + + from . import RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT + from ._util import (validate_rss_token, generate_watch_guid, get_rss_template, + get_watch_label, build_notification_context, render_notification, + populate_feed_entry, add_watch_categories) + from ...notification_service import NotificationService + """ Display an RSS feed for all unviewed watches that belong to a specific tag. Returns RSS XML with entries for each unviewed watch with sufficient history. """ - # Always requires token set - app_rss_token = datastore.data['settings']['application'].get('rss_access_token') - rss_url_token = request.args.get('token') - rss_content_format = datastore.data['settings']['application'].get('rss_content_format') + # Validate token + is_valid, error = validate_rss_token(datastore, request) + if not is_valid: + return error - if rss_url_token != app_rss_token: - return "Access denied, bad token", 403 + rss_content_format = datastore.data['settings']['application'].get('rss_content_format') # Verify tag exists tag = datastore.data['settings']['application'].get('tags', {}).get(tag_uuid) @@ -42,9 +42,12 @@ def construct_tag_routes(rss_blueprint, datastore): fg.title(f'changedetection.io - {tag_title}') fg.description(f'Changes for watches tagged with {tag_title}') fg.link(href='https://changedetection.io') - + notification_service = NotificationService(datastore=datastore, notification_q=False) # Find all watches with this tag for uuid, watch in datastore.data['watching'].items(): + #@todo This is wrong, it needs to sort by most recently changed and then limit it datastore.data['watching'].items().sorted(?) + # So get all watches in this tag then sort + # Skip if watch doesn't have this tag if tag_uuid not in watch.get('tags', []): continue @@ -63,31 +66,32 @@ def construct_tag_routes(rss_blueprint, datastore): # Add uuid to watch for proper functioning watch['uuid'] = uuid - # Generate GUID for this entry - guid = generate_watch_guid(watch) - fe = fg.add_entry() - # Include a link to the diff page diff_link = {'href': url_for('ui.ui_views.diff_history_page', uuid=watch['uuid'], _external=True)} - fe.link(link=diff_link) - # Generate diff content - content, watch_label = generate_watch_diff_content(watch, dates, rss_content_format, datastore) + # Get watch label + watch_label = get_watch_label(datastore, watch) - fe.title(title=watch_label) - fe.content(content=content, type='CDATA') - fe.guid(guid, permalink=False) - dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key)) - dt = dt.replace(tzinfo=pytz.UTC) - fe.pubDate(dt) + # Get template and build notification context + timestamp_to = dates[-1] + timestamp_from = dates[-2] - # Add categories based on watch tags - for tag_uuid in watch.get('tags', []): - tag = datastore.data['settings']['application'].get('tags', {}).get(tag_uuid) - if tag: - tag_title = tag.get('title', '') - if tag_title: - fe.category(term=tag_title) + # Generate GUID for this entry + guid = generate_watch_guid(watch, timestamp_to) + n_body_template = get_rss_template(datastore, watch, rss_content_format, + RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT) + + n_object = build_notification_context(watch, timestamp_from, timestamp_to, + watch_label, n_body_template, rss_content_format) + + # Render notification + res = render_notification(n_object, notification_service, watch, datastore) + + # Create and populate feed entry + fe = fg.add_entry() + title_suffix = f"Change @ {res['original_context']['change_datetime']}" + populate_feed_entry(fe, watch, res['body'], guid, timestamp_to, link=diff_link, title_suffix=title_suffix) + add_watch_categories(fe, watch, datastore) response = make_response(fg.rss_str()) response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8') diff --git a/changedetectionio/blueprint/settings/templates/settings.html b/changedetectionio/blueprint/settings/templates/settings.html index 6d3376ca..ce13ef00 100644 --- a/changedetectionio/blueprint/settings/templates/settings.html +++ b/changedetectionio/blueprint/settings/templates/settings.html @@ -2,7 +2,7 @@ {% block content %} {% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, render_ternary_field, render_fieldlist_with_inline_errors %} -{% from '_common_fields.html' import render_common_settings_form %} +{% from '_common_fields.html' import render_common_settings_form, show_token_placeholders %}