diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 7920383a1..32bbe708c 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -11,6 +11,4 @@ updates:
- package-ecosystem: pip
directory: /
schedule:
- interval: "daily"
- allow:
- - dependency-name: "apprise"
+ interval: "weekly"
diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py
index 0d48769a7..255322c0e 100644
--- a/changedetectionio/__init__.py
+++ b/changedetectionio/__init__.py
@@ -2,7 +2,7 @@
# Read more https://github.com/dgtlmoon/changedetection.io/wiki
-__version__ = '0.50.14'
+__version__ = '0.50.17'
from changedetectionio.strtobool import strtobool
from json.decoder import JSONDecodeError
diff --git a/changedetectionio/processors/magic.py b/changedetectionio/processors/magic.py
new file mode 100644
index 000000000..bdfda2ded
--- /dev/null
+++ b/changedetectionio/processors/magic.py
@@ -0,0 +1,138 @@
+"""
+Content Type Detection and Stream Classification
+
+This module provides intelligent content-type detection for changedetection.io.
+It addresses the common problem where HTTP Content-Type headers are missing, incorrect,
+or too generic, which would otherwise cause the wrong processor to be used.
+
+The guess_stream_type class combines:
+1. HTTP Content-Type headers (when available and reliable)
+2. Python-magic library for MIME detection (analyzing actual file content)
+3. Content-based pattern matching for text formats (HTML tags, XML declarations, etc.)
+
+This multi-layered approach ensures accurate detection of RSS feeds, JSON, HTML, PDF,
+plain text, CSV, YAML, and XML formats - even when servers provide misleading headers.
+
+Used by: processors/text_json_diff/processor.py and other content processors
+"""
+
+# When to apply the 'cdata to real HTML' hack
+RSS_XML_CONTENT_TYPES = [
+ "application/rss+xml",
+ "application/rdf+xml",
+ "text/xml",
+ "application/xml",
+ "application/atom+xml",
+ "text/rss+xml", # rare, non-standard
+ "application/x-rss+xml", # legacy (older feed software)
+ "application/x-atom+xml", # legacy (older Atom)
+]
+
+# JSON Content-types
+JSON_CONTENT_TYPES = [
+ "application/activity+json",
+ "application/feed+json",
+ "application/json",
+ "application/ld+json",
+ "application/vnd.api+json",
+]
+
+# CSV Content-types
+CSV_CONTENT_TYPES = [
+ "text/csv",
+ "application/csv",
+]
+
+# Generic XML Content-types (non-RSS/Atom)
+XML_CONTENT_TYPES = [
+ "text/xml",
+ "application/xml",
+]
+
+# YAML Content-types
+YAML_CONTENT_TYPES = [
+ "text/yaml",
+ "text/x-yaml",
+ "application/yaml",
+ "application/x-yaml",
+]
+
+HTML_PATTERNS = ['
+
+ :param client:
+ :param live_server:
+ :param measure_memory_usage:
+ :return:
+ """
+ with open("test-datastore/endpoint-content.txt", "w") as f:
+ f.write("""some random text that should be split by line
+and not parsed with html_to_text
+Even this title should stay because we are just plain text
+this way we know that it correctly parsed as plain text
+\r\n
+ok\r\n
+got it\r\n
+""")
+
+ test_url = url_for('test_endpoint', content_type="text/plain", _external=True)
+
+ # Add our URL to the import page
+ res = client.post(
+ url_for("imports.import_page"),
+ data={"urls": test_url},
+ follow_redirects=True
+ )
+
+ assert b"1 Imported" in res.data
+
+ wait_for_all_checks(client)
+
+ ### check the front end
+ res = client.get(
+ url_for("ui.ui_views.preview_page", uuid="first"),
+ follow_redirects=True
+ )
+
+ assert b"some random text that should be split by line\n" in res.data
+ ####
+
+ # Check the snapshot by API that it has linefeeds too
+ watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+ api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+ res = client.get(
+ url_for("watchhistory", uuid=watch_uuid),
+ headers={'x-api-key': api_key},
+ )
+
+ # Fetch a snapshot by timestamp, check the right one was found
+ res = client.get(
+ url_for("watchsinglehistory", uuid=watch_uuid, timestamp=list(res.json.keys())[-1]),
+ headers={'x-api-key': api_key},
+ )
+ assert b"some random text that should be split by line\n" in res.data
+ assert b"Even this title should stay because we are just plain text" in res.data
+
+ res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
+
diff --git a/changedetectionio/tests/test_group.py b/changedetectionio/tests/test_group.py
index 5e2596c54..e63639a9d 100644
--- a/changedetectionio/tests/test_group.py
+++ b/changedetectionio/tests/test_group.py
@@ -264,8 +264,6 @@ def test_limit_tag_ui(client, live_server, measure_memory_usage):
client.get(url_for('ui.mark_all_viewed', tag=tag_uuid), follow_redirects=True)
wait_for_all_checks(client)
- with open('/tmp/fuck.html', 'wb') as f:
- f.write(res.data)
# Should be only 1 unviewed
res = client.get(url_for("watchlist.index"))
assert res.data.count(b' unviewed ') == 1
diff --git a/changedetectionio/tests/test_history_consistency.py b/changedetectionio/tests/test_history_consistency.py
index a16e99f73..b8a21cf2c 100644
--- a/changedetectionio/tests/test_history_consistency.py
+++ b/changedetectionio/tests/test_history_consistency.py
@@ -3,9 +3,8 @@
import time
import os
import json
-import logging
from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from .util import wait_for_all_checks
from urllib.parse import urlparse, parse_qs
def test_consistent_history(client, live_server, measure_memory_usage):
diff --git a/changedetectionio/tests/test_xpath_selector.py b/changedetectionio/tests/test_xpath_selector.py
index d79fa4b3c..abcc766a2 100644
--- a/changedetectionio/tests/test_xpath_selector.py
+++ b/changedetectionio/tests/test_xpath_selector.py
@@ -1,12 +1,42 @@
# -*- coding: utf-8 -*-
-import time
+
from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
-
-from ..html_tools import *
+from .util import wait_for_all_checks
+from ..processors.magic import RSS_XML_CONTENT_TYPES
+def set_rss_atom_feed_response(header=''):
+ test_return_data = f"""{header}
+
+
+
+ RSS Feed
+
+
+
+
+
+
+ en-us
+ water News RSS
+ -
+ 🍁 Lets go discount
+
ok heres the description
+
+
+
+ Wed, 08 Oct 2025 15:28:55 +0000
+ https://store.waterpowered.com/news/app/1643320/view/511845698831908921
+
+
+
+"""
+
+ with open("test-datastore/endpoint-content.txt", "w") as f:
+ f.write(test_return_data)
+
+ return None
@@ -575,3 +605,47 @@ def test_xpath_20_function_string_join_matches(client, live_server, measure_memo
client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
+
+def _subtest_xpath_rss(client, content_type='text/html'):
+
+ # Add our URL to the import page
+ test_url = url_for('test_endpoint', content_type=content_type, _external=True)
+ res = client.post(
+ url_for("ui.ui_views.form_quick_watch_add"),
+ data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+ follow_redirects=True
+ )
+
+ assert b"Watch added in Paused state, saving will unpause" in res.data
+
+ res = client.post(
+ url_for("ui.ui_edit.edit_page", uuid="first", unpause_on_save=1),
+ data={
+ "url": test_url,
+ "include_filters": "xpath://item",
+ "tags": '',
+ "fetch_backend": "html_requests",
+ "time_between_check_use_default": "y",
+ },
+ follow_redirects=True
+ )
+
+ assert b"unpaused" in res.data
+ wait_for_all_checks(client)
+
+ res = client.get(
+ url_for("ui.ui_views.preview_page", uuid="first"),
+ follow_redirects=True
+ )
+
+ assert b"Lets go discount" in res.data, f"When testing for Lets go discount called with content type '{content_type}'"
+ assert b"Events and Announcements" not in res.data, f"When testing for Lets go discount called with content type '{content_type}'" # It should not be here because thats not our selector target
+
+ client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
+
+# Be sure all-in-the-wild types of RSS feeds work with xpath
+def test_rss_xpath(client, live_server):
+ for feed_header in ['', '']:
+ set_rss_atom_feed_response(header=feed_header)
+ for content_type in RSS_XML_CONTENT_TYPES:
+ _subtest_xpath_rss(client, content_type=content_type)
diff --git a/requirements.txt b/requirements.txt
index d448f7aa4..cf95181d3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -135,7 +135,7 @@ tzdata
pluggy ~= 1.5
# Needed for testing, cross-platform for process and system monitoring
-psutil==7.0.0
+psutil==7.1.0
ruff >= 0.11.2
pre_commit >= 4.2.0