From c3361943aad38d6746a746451e96f043e2ef230b Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sat, 15 Aug 2026 18:59:59 +0200 Subject: [PATCH] Improved content type detection - adding regression test --- .../tests/unit/test_content_type_detection.py | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 changedetectionio/tests/unit/test_content_type_detection.py diff --git a/changedetectionio/tests/unit/test_content_type_detection.py b/changedetectionio/tests/unit/test_content_type_detection.py new file mode 100644 index 00000000..e5746bb0 --- /dev/null +++ b/changedetectionio/tests/unit/test_content_type_detection.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# run from dir above changedetectionio/ dir +# python3 -m unittest changedetectionio.tests.unit.test_content_type_detection + +"""Unit tests for guess_stream_type() content-type classification. + +Regression cover for #4302: a normal HTML page served as 'text/html;charset=utf-8' was +classified as plaintext, which made processor.py skip html_to_text() entirely and store the +raw filtered block as the snapshot (visible as literal '
' separators in every diff). + +Two independent failures had to line up: + 1. The header test was an exact comparison ("text/html"), so any charset parameter missed it. + 2. has_html_patterns only sniffs content[:200], and the page put 241 bytes of HTML comments + before is_plaintext. +""" + +import unittest + +from changedetectionio.processors.magic import guess_stream_type + +# Real-world shape from https://www.newbalancemexico.com/c/lo-mas-nuevo (Salesforce/ISML templates): +# blank lines and two long comments push \n" + "\n\n\n" + '\n\n\n\n\n' + '
' +) + +PLAIN_HTML = 't

hi

' + + +class TestGuessStreamType(unittest.TestCase): + + def _flags(self, header, content): + st = guess_stream_type(http_content_header=header, content=content) + return {n: getattr(st, n) for n in + ('is_html', 'is_plaintext', 'is_rss', 'is_xml', 'is_json', 'is_pdf', 'is_csv')} + + def test_preamble_pushes_doctype_past_sniff_window(self): + # Guard the premise of the test below - if this ever shrinks, the regression case is no + # longer exercising the "content sniff cannot help" path. + self.assertGreater(COMMENT_PREAMBLE_HTML.index('\nf' + atom = '\nf' + json_doc = '{"title": "hello", "items": [1, 2, 3]}' + + self.assertTrue(self._flags('text/html;charset=utf-8', rss)['is_rss']) + self.assertTrue(self._flags('text/html;charset=utf-8', atom)['is_rss']) + self.assertTrue(self._flags('application/json', json_doc)['is_json']) + self.assertTrue(self._flags('text/xml', 'x')['is_xml']) + + +class TestFilterSeparatorSurvivesTextExtraction(unittest.TestCase): + """#4302 end to end: an attribute xpath must produce newlines, never a literal '
'. + + xpath_filter injects TEXT_FILTER_LIST_LINE_SUFFIX ('
') between matches that have no + .tag attribute - i.e. attribute and text() nodes, which is precisely when the filter output + is no longer HTML. That marker is only removed if html_to_text() runs, so this test pins the + classification and the conversion together. + """ + + def test_attribute_xpath_matches_become_newlines(self): + from changedetectionio import html_tools + + filtered = html_tools.xpath_filter( + xpath_filter='//div[@class="product"]/@data-pid', + html_content=COMMENT_PREAMBLE_HTML.replace( + '
', + '
' + '
'), + append_pretty_line_formatting=True, + ) + # The marker is expected at this stage - it is an intermediate value, not a snapshot. + self.assertIn('
', filtered) + + stream = guess_stream_type(http_content_header='text/html;charset=utf-8', + content=COMMENT_PREAMBLE_HTML) + self.assertTrue(stream.is_html) + self.assertFalse(stream.is_plaintext) + + text = html_tools.html_to_text(html_content=filtered, is_rss=stream.is_rss) + self.assertNotIn('
', text, "'
' leaked into the snapshot - see #4302") + self.assertIn('U204LMMA', text) + self.assertIn('MR530KA', text) + self.assertEqual(['U204LMMA', 'MR530KA'], text.split()) + + +if __name__ == '__main__': + # Can run this file directly for quick testing + unittest.main()