lee/changedetection.io
1
0
Fork 0
mirror of https://github.com/dgtlmoon/changedetection.io.git synced 2025-12-10 10:05:33 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Always follow plaintext header over the actual content type if its available (#3473) #3472

Browse Source
This commit is contained in:
dgtlmoon
2025-10-09 12:56:32 +02:00
committed by GitHub
parent 0970c087c8
commit 3e59521f48
3 changed files with 47 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
changedetectionio/processors/magic.py
View File

@@ -20,8 +20,6 @@ Used by: processors/text_json_diff/processor.py and other content processors
RSS_XML_CONTENT_TYPES = [ RSS_XML_CONTENT_TYPES = [
"application/rss+xml", "application/rss+xml",
"application/rdf+xml", "application/rdf+xml",
"text/xml",
"application/xml",
"application/atom+xml", "application/atom+xml",
"text/rss+xml", # rare, non-standard "text/rss+xml", # rare, non-standard
"application/x-rss+xml", # legacy (older feed software) "application/x-rss+xml", # legacy (older feed software)
@@ -37,11 +35,6 @@ JSON_CONTENT_TYPES = [
"application/vnd.api+json", "application/vnd.api+json",
] ]
# CSV Content-types
CSV_CONTENT_TYPES = [
"text/csv",
"application/csv",
]
# Generic XML Content-types (non-RSS/Atom) # Generic XML Content-types (non-RSS/Atom)
XML_CONTENT_TYPES = [ XML_CONTENT_TYPES = [
@@ -49,14 +42,6 @@ XML_CONTENT_TYPES = [
"application/xml", "application/xml",
] ]
# YAML Content-types
YAML_CONTENT_TYPES = [
"text/yaml",
"text/x-yaml",
"application/yaml",
"application/x-yaml",
]
HTML_PATTERNS = ['<!doctype html', '<html', '<head', '<body', '<script', '<iframe', '<div'] HTML_PATTERNS = ['<!doctype html', '<html', '<head', '<body', '<script', '<iframe', '<div']
import re import re
@@ -104,18 +89,16 @@ class guess_stream_type():
has_html_patterns = any(p in test_content_normalized for p in HTML_PATTERNS) has_html_patterns = any(p in test_content_normalized for p in HTML_PATTERNS)
# Always trust headers first # Always trust headers first
if any(s in http_content_header for s in RSS_XML_CONTENT_TYPES) or any(s in magic_content_header for s in RSS_XML_CONTENT_TYPES): if 'text/plain' in http_content_header:
self.is_plaintext = True
if any(s in http_content_header for s in RSS_XML_CONTENT_TYPES):
self.is_rss = True self.is_rss = True
elif any(s in http_content_header for s in JSON_CONTENT_TYPES) or any(s in magic_content_header for s in JSON_CONTENT_TYPES): elif any(s in http_content_header for s in JSON_CONTENT_TYPES):
self.is_json = True self.is_json = True
elif any(s in http_content_header for s in CSV_CONTENT_TYPES) or any(s in magic_content_header for s in CSV_CONTENT_TYPES): elif any(s in http_content_header for s in XML_CONTENT_TYPES):
self.is_csv = True
elif any(s in http_content_header for s in XML_CONTENT_TYPES) or any(s in magic_content_header for s in XML_CONTENT_TYPES):
# Only mark as generic XML if not already detected as RSS # Only mark as generic XML if not already detected as RSS
if not self.is_rss: if not self.is_rss:
self.is_xml = True self.is_xml = True
elif any(s in http_content_header for s in YAML_CONTENT_TYPES) or any(s in magic_content_header for s in YAML_CONTENT_TYPES):
self.is_yaml = True
elif 'pdf' in magic_content_header: elif 'pdf' in magic_content_header:
self.is_pdf = True self.is_pdf = True
### ###
@@ -125,13 +108,18 @@ class guess_stream_type():
elif magic_result == 'text/plain': elif magic_result == 'text/plain':
self.is_plaintext = True self.is_plaintext = True
logger.debug(f"Trusting magic's text/plain result (no HTML patterns detected)") logger.debug(f"Trusting magic's text/plain result (no HTML patterns detected)")
elif '<rss' in test_content_normalized or '<feed' in test_content_normalized: elif any(s in magic_content_header for s in JSON_CONTENT_TYPES):
self.is_json = True
# magic will call a rss document 'xml'
elif '<rss' in test_content_normalized or '<feed' in test_content_normalized or any(s in magic_content_header for s in RSS_XML_CONTENT_TYPES):
self.is_rss = True self.is_rss = True
elif test_content_normalized.startswith('<?xml'): elif test_content_normalized.startswith('<?xml') or any(s in magic_content_header for s in XML_CONTENT_TYPES):
# Generic XML that's not RSS/Atom (RSS/Atom checked above) # Generic XML that's not RSS/Atom (RSS/Atom checked above)
self.is_xml = True self.is_xml = True
elif '%pdf-1' in test_content: elif '%pdf-1' in test_content:
self.is_pdf = True self.is_pdf = True
elif http_content_header.startswith('text/'):
self.is_plaintext = True
# Only trust magic for 'text' if no other patterns matched # Only trust magic for 'text' if no other patterns matched
elif 'text' in magic_content_header: elif 'text' in magic_content_header:
self.is_plaintext = True self.is_plaintext = True

33
changedetectionio/tests/test_backend.py
View File

@@ -295,3 +295,36 @@ got it\r\n
res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
# Server says its plaintext, we should always treat it as plaintext
def test_plaintext_even_if_xml_content(client, live_server, measure_memory_usage):
with open("test-datastore/endpoint-content.txt", "w") as f:
f.write("""<?xml version="1.0" encoding="utf-8"?>
<resources xmlns:tools="http://schemas.android.com/tools">
<!--Activity and fragment titles-->
<string name="feed_update_receiver_name">Abonnementen bijwerken</string>
</resources>
""")
test_url = url_for('test_endpoint', content_type="text/plain", _external=True)
# Add our URL to the import page
res = client.post(
url_for("imports.import_page"),
data={"urls": test_url},
follow_redirects=True
)
assert b"1 Imported" in res.data
wait_for_all_checks(client)
res = client.get(
url_for("ui.ui_views.preview_page", uuid="first"),
follow_redirects=True
)
assert b'&lt;string name=&#34;feed_update_receiver_name&#34;' in res.data
res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)

4
changedetectionio/tests/test_rss.py
View File

@@ -111,7 +111,7 @@ def test_basic_cdata_rss_markup(client, live_server, measure_memory_usage):
set_original_cdata_xml() set_original_cdata_xml()
test_url = url_for('test_endpoint', content_type="application/xml", _external=True) test_url = url_for('test_endpoint', content_type="application/atom+xml; charset=UTF-8", _external=True)
# Add our URL to the import page # Add our URL to the import page
res = client.post( res = client.post(
@@ -139,7 +139,7 @@ def test_rss_xpath_filtering(client, live_server, measure_memory_usage):
set_original_cdata_xml() set_original_cdata_xml()
test_url = url_for('test_endpoint', content_type="application/xml", _external=True) test_url = url_for('test_endpoint', content_type="application/atom+xml; charset=UTF-8", _external=True)
res = client.post( res = client.post(
url_for("ui.ui_views.form_quick_watch_add"), url_for("ui.ui_views.form_quick_watch_add"),