diff --git a/changedetectionio/content_fetchers/requests.py b/changedetectionio/content_fetchers/requests.py index 514883b7..f5b9d51e 100644 --- a/changedetectionio/content_fetchers/requests.py +++ b/changedetectionio/content_fetchers/requests.py @@ -1,6 +1,7 @@ from loguru import logger import hashlib import os +import re import asyncio from changedetectionio import strtobool from changedetectionio.content_fetchers.exceptions import BrowserStepsInUnsupportedFetcher, EmptyReply, Non200ErrorCodeReceived @@ -76,9 +77,22 @@ class fetcher(Fetcher): if not is_binary: # Don't run this for PDF (and requests identified as binary) takes a _long_ time if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'): - encoding = chardet.detect(r.content)['encoding'] - if encoding: - r.encoding = encoding + # For XML/RSS feeds, check the XML declaration for encoding attribute + # This is more reliable than chardet which can misdetect UTF-8 as MacRoman + content_type = r.headers.get('content-type', '').lower() + if 'xml' in content_type or 'rss' in content_type: + # Look for + xml_encoding_match = re.search(rb'<\?xml[^>]+encoding=["\']([^"\']+)["\']', r.content[:200]) + if xml_encoding_match: + r.encoding = xml_encoding_match.group(1).decode('ascii') + else: + # Default to UTF-8 for XML if no encoding found + r.encoding = 'utf-8' + else: + # For other content types, use chardet + encoding = chardet.detect(r.content)['encoding'] + if encoding: + r.encoding = encoding self.headers = r.headers diff --git a/changedetectionio/rss_tools.py b/changedetectionio/rss_tools.py index e5a2fb5c..f3d3eac7 100644 --- a/changedetectionio/rss_tools.py +++ b/changedetectionio/rss_tools.py @@ -29,16 +29,135 @@ def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False return re.sub(pattern, repl, html_content) +# Jinja2 template for formatting RSS/Atom feed entries +# Covers all common feedparser entry fields including namespaced elements +# Outputs HTML that will be converted to text via html_to_text +# @todo - This could be a UI setting in the future +RSS_ENTRY_TEMPLATE = """
{%- if entry.title -%}Title: {{ entry.title }}
{%- endif -%} +{%- if entry.link -%}Link: {{ entry.link }}
+{%- endif -%} +{%- if entry.id -%} +Guid: {{ entry.id }}
+{%- endif -%} +{%- if entry.published -%} +PubDate: {{ entry.published }}
+{%- endif -%} +{%- if entry.updated and entry.updated != entry.published -%} +Updated: {{ entry.updated }}
+{%- endif -%} +{%- if entry.author -%} +Author: {{ entry.author }}
+{%- elif entry.author_detail and entry.author_detail.name -%} +Author: {{ entry.author_detail.name }} +{%- if entry.author_detail.email %} ({{ entry.author_detail.email }}){% endif -%} +
+{%- endif -%} +{%- if entry.contributors -%} +Contributors: {% for contributor in entry.contributors -%} +{{ contributor.name if contributor.name else contributor }} +{%- if not loop.last %}, {% endif -%} +{%- endfor %}
+{%- endif -%} +{%- if entry.publisher -%} +Publisher: {{ entry.publisher }}
+{%- endif -%} +{%- if entry.rights -%} +Rights: {{ entry.rights }}
+{%- endif -%} +{%- if entry.license -%} +License: {{ entry.license }}
+{%- endif -%} +{%- if entry.language -%} +Language: {{ entry.language }}
+{%- endif -%} +{%- if entry.tags -%} +Tags: {% for tag in entry.tags -%} +{{ tag.term if tag.term else tag }} +{%- if not loop.last %}, {% endif -%} +{%- endfor %}
+{%- endif -%} +{%- if entry.category -%} +Category: {{ entry.category }}
+{%- endif -%} +{%- if entry.comments -%} +Comments: {{ entry.comments }}
+{%- endif -%} +{%- if entry.slash_comments -%} +Comment Count: {{ entry.slash_comments }}
+{%- endif -%} +{%- if entry.enclosures -%} +Enclosures:
+{%- for enclosure in entry.enclosures %} +- {{ enclosure.href }} ({{ enclosure.type if enclosure.type else 'unknown type' }} +{%- if enclosure.length %}, {{ enclosure.length }} bytes{% endif -%} +)
+{%- endfor -%} +{%- endif -%} +{%- if entry.media_content -%} +Media:
+{%- for media in entry.media_content %} +- {{ media.url }} +{%- if media.type %} ({{ media.type }}){% endif -%} +{%- if media.width and media.height %} {{ media.width }}x{{ media.height }}{% endif -%} +
+{%- endfor -%} +{%- endif -%} +{%- if entry.media_thumbnail -%} +Thumbnail: {{ entry.media_thumbnail[0].url if entry.media_thumbnail[0].url else entry.media_thumbnail[0] }}
+{%- endif -%} +{%- if entry.media_description -%} +Media Description: {{ entry.media_description }}
+{%- endif -%} +{%- if entry.itunes_duration -%} +Duration: {{ entry.itunes_duration }}
+{%- endif -%} +{%- if entry.itunes_author -%} +Podcast Author: {{ entry.itunes_author }}
+{%- endif -%} +{%- if entry.dc_identifier -%} +Identifier: {{ entry.dc_identifier }}
+{%- endif -%} +{%- if entry.dc_source -%} +DC Source: {{ entry.dc_source }}
+{%- endif -%} +{%- if entry.dc_type -%} +Type: {{ entry.dc_type }}
+{%- endif -%} +{%- if entry.dc_format -%} +Format: {{ entry.dc_format }}
+{%- endif -%} +{%- if entry.dc_relation -%} +Related: {{ entry.dc_relation }}
+{%- endif -%} +{%- if entry.dc_coverage -%} +Coverage: {{ entry.dc_coverage }}
+{%- endif -%} +{%- if entry.source and entry.source.title -%} +Source: {{ entry.source.title }} +{%- if entry.source.link %} ({{ entry.source.link }}){% endif -%} +
+{%- endif -%} +{%- if entry.dc_content -%} +Content: {{ entry.dc_content | safe }} +{%- elif entry.content and entry.content[0].value -%} +Content: {{ entry.content[0].value | safe }} +{%- elif entry.summary -%} +Summary: {{ entry.summary | safe }} +{%- endif -%}
+""" + + def format_rss_items(rss_content: str, render_anchor_tag_content=False) -> str: """ - Format RSS/Atom feed items in a readable text format using feedparser. + Format RSS/Atom feed items in a readable text format using feedparser and Jinja2. - Converts RSS or Atom elements to formatted text with: - - → <h1>Title</h1> - - <link> → Link: [url] - - <guid> → Guid: [id] - - <pubDate> → PubDate: [date] - - <description> or <content> → Raw HTML content (CDATA and entities automatically handled) + Converts RSS <item> or Atom <entry> elements to formatted text with all available fields: + - Basic fields: title, link, id/guid, published date, updated date + - Author fields: author, author_detail, contributors, publisher + - Content fields: content, summary, description + - Metadata: tags, category, rights, license + - Media: enclosures, media_content, media_thumbnail + - Dublin Core elements: dc:creator, dc:date, dc:publisher, etc. (mapped by feedparser) Args: rss_content: The RSS/Atom feed content @@ -49,65 +168,19 @@ def format_rss_items(rss_content: str, render_anchor_tag_content=False) -> str: """ try: import feedparser - from xml.sax.saxutils import escape as xml_escape + from changedetectionio.jinja2_custom import safe_jinja # Parse the feed - feedparser handles all RSS/Atom variants, CDATA, entity unescaping, etc. feed = feedparser.parse(rss_content) - formatted_items = [] - - # Determine feed type for appropriate labels when fields are missing - # feedparser sets feed.version to things like 'rss20', 'atom10', etc. + # Determine feed type for appropriate labels is_atom = feed.version and 'atom' in feed.version + formatted_items = [] for entry in feed.entries: - item_parts = [] - - # Title - feedparser handles CDATA and entity unescaping automatically - if hasattr(entry, 'title') and entry.title: - item_parts.append(f'<h1>{xml_escape(entry.title)}</h1>') - - # Link - if hasattr(entry, 'link') and entry.link: - item_parts.append(f'Link: {xml_escape(entry.link)}<br>') - - # GUID/ID - if hasattr(entry, 'id') and entry.id: - item_parts.append(f'Guid: {xml_escape(entry.id)}<br>') - - # Date - feedparser normalizes all date field names to 'published' - if hasattr(entry, 'published') and entry.published: - item_parts.append(f'PubDate: {xml_escape(entry.published)}<br>') - - # Description/Content - feedparser handles CDATA and entity unescaping automatically - # Only add "Summary:" label for Atom <summary> tags - content = None - add_label = False - - if hasattr(entry, 'content') and entry.content: - # Atom <content> - no label, just content - content = entry.content[0].value if entry.content[0].value else None - elif hasattr(entry, 'summary'): - # Could be RSS <description> or Atom <summary> - # feedparser maps both to entry.summary - content = entry.summary if entry.summary else None - # Only add "Summary:" label for Atom feeds (which use <summary> tag) - if is_atom: - add_label = True - - # Add content with or without label - if content: - if add_label: - item_parts.append(f'Summary:<br>{content}') - else: - item_parts.append(content) - else: - # No content - just show <none> - item_parts.append('<none>') - - # Join all parts of this item - if item_parts: - formatted_items.append('\n'.join(item_parts)) + # Render the entry using Jinja2 template + rendered = safe_jinja.render(RSS_ENTRY_TEMPLATE, entry=entry, is_atom=is_atom) + formatted_items.append(rendered.strip()) # Wrap each item in a div with classes (first, last, item-N) items_html = [] @@ -122,7 +195,8 @@ def format_rss_items(rss_content: str, render_anchor_tag_content=False) -> str: class_str = ' '.join(classes) items_html.append(f'<div class="{class_str}">{item}</div>') - return '<html><body>\n'+"\n<br><br>".join(items_html)+'\n</body></html>' + + return '<html><body>\n' + "\n<br>".join(items_html) + '\n</body></html>' except Exception as e: logger.warning(f"Error formatting RSS items: {str(e)}") diff --git a/changedetectionio/tests/test_rss_reader_mode.py b/changedetectionio/tests/test_rss_reader_mode.py index e42e6cc1..63736052 100644 --- a/changedetectionio/tests/test_rss_reader_mode.py +++ b/changedetectionio/tests/test_rss_reader_mode.py @@ -7,6 +7,61 @@ from flask import url_for from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \ extract_UUID_from_client, delete_all_watches +def set_xmlns_purl_content(datastore_path, extra=""): + data=f"""<rss xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:dc="https://purl.org/dc/elements/1.1/" xmlns:media="http://search.yahoo.com/mrss/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0"> +<channel> +<atom:link href="https://www.xxxxxxxtechxxxxx.com/feeds.xml" rel="self" type="application/rss+xml"/> +<title> +<![CDATA[ Latest from xxxxxxxtechxxxxx ]]> + +https://www.xxxxx.com + + + +Wed, 19 Nov 2025 15:00:00 +0000 +en + + +<![CDATA[ Sony Xperia 1 VII review: has Sony’s long-standing Xperia family lost what it takes to compete? ]]> + + +

On the plus side, you don't technically need to solve the final one, as you'll be able to answer that one by a process of elimination. What's more, you can make up to four mistakes, which gives you a little bit of breathing room.

It's a little more involved than something like Wordle, however, and there are plenty of opportunities for the game to trip you up with tricks. For instance, watch out for homophones and other word games that could disguise the answers.

It's playable for free via the NYT Games site on desktop or mobile.

]]> + +https://www.xxxxxxx.com/gaming/nyt-connections-today-answers-hints-20-november-2025 + + + +N2C2T6DztpWdxSdKpSUx89 + +Wed, 19 Nov 2025 15:00:00 +0000 + + + + + + + + + + + + + + + + + + + + + + """ + + with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f: + f.write(data) + + + def set_original_cdata_xml(datastore_path): test_return_data = """ @@ -98,3 +153,26 @@ def test_rss_reader_mode_with_css_filters(client, live_server, measure_memory_us assert 'The days of Terminator and The Matrix' in snapshot_contents delete_all_watches(client) + +def test_xmlns_purl_content(client, live_server, measure_memory_usage, datastore_path): + set_xmlns_purl_content(datastore_path=datastore_path) + + # Rarely do endpoints give the right header, usually just text/xml, so we check also for