diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index 8e8f12eac..cdf4fa2a6 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -757,16 +757,41 @@ def get_triggered_text(content, trigger_text): def extract_title(data: bytes | str, sniff_bytes: int = 2048, scan_chars: int = 8192) -> str | None: + """Extract the from an HTML document. + + Rather than decoding/scanning a fixed prefix of the whole document, we first + locate the raw ``<title`` marker and then decode only a small window around + it. This handles pages (e.g. Amazon) where large ``<head>`` sections push + the title tag well past the old 8 192-character scan limit. + """ + # Maximum bytes/chars to extract after (and including) the opening <title tag. + # The regex needs to see , so the window must cover the full content. + # The return value is always capped at 2 000 chars; titles beyond that are + # rare but possible. We read up to 128 KiB from the tag onwards to handle + # even pathological cases without scanning the whole document. + _TITLE_WINDOW = 131072 + try: - # Only decode/process the prefix we need for title extraction match data: - case bytes() if data.startswith((b"\xff\xfe", b"\xfe\xff")): - prefix = data[:scan_chars * 2].decode("utf-16", errors="replace") case bytes() if data.startswith((b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff")): - prefix = data[:scan_chars * 4].decode("utf-32", errors="replace") + # UTF-32: locate the tag in the raw bytes, then decode the window. + tag_pos = data.lower().find(b"<\x00\x00\x00t\x00\x00\x00") + if tag_pos == -1: + return None + chunk = data[tag_pos: tag_pos + _TITLE_WINDOW * 4].decode("utf-32", errors="replace") + prefix = chunk + case bytes() if data.startswith((b"\xff\xfe", b"\xfe\xff")): + # UTF-16: simple byte-pair search is tricky; fall back to decoding + # a reasonable head chunk and let the regex do the rest. + prefix = data[: max(scan_chars * 2, _TITLE_WINDOW)].decode("utf-16", errors="replace") case bytes(): + # UTF-8 / legacy 8-bit: find the tag cheaply in raw bytes. + tag_pos = data.lower().find(b" scan_chars else data + tag_pos = data.lower().find(" is +pushed past the hard-coded 8 192-character scan window by large content +(e.g. Amazon product pages where can sit at character index 55 000+). +""" + +import unittest + +from changedetectionio.html_tools import extract_title + + +def _make_large_head_page(title: str, filler_count: int = 500) -> bytes: + """Build a synthetic HTML page whose <title> is pushed far past 8 192 chars. + + Each filler line is ~126 bytes; 500 lines ≈ 63 000 bytes before <title>. + """ + filler_line = '<meta name="x" content="' + "A" * 100 + '"/>\n' + head_junk = filler_line * filler_count + page = ( + f"<html><head>{head_junk}" + f"<title>{title}" + f"" + ) + return page.encode("utf-8") + + +class TestExtractTitle(unittest.TestCase): + # ------------------------------------------------------------------ + # Regression: issue #4217 — large pushes past scan limit + # ------------------------------------------------------------------ + + def test_large_head_bytes_title_extracted(self): + """<title> beyond 8 192 bytes must still be extracted (bytes input).""" + page = _make_large_head_page("Amazon Product Title - Real Title Here") + title_pos = page.find(b"<title") + self.assertGreater( + title_pos, + 8192, + f"Precondition: <title> must be past 8 192 chars (actual: {title_pos})", + ) + result = extract_title(page) + self.assertEqual(result, "Amazon Product Title - Real Title Here") + + def test_large_head_str_title_extracted(self): + """<title> beyond 8 192 chars must still be extracted (str input).""" + page_bytes = _make_large_head_page("Large Head String Test") + page_str = page_bytes.decode("utf-8") + title_pos = page_str.find("<title") + self.assertGreater(title_pos, 8192) + result = extract_title(page_str) + self.assertEqual(result, "Large Head String Test") + + def test_very_large_head_55000_chars(self): + """Simulate Amazon-like pages where <title> is at ~55 000 chars.""" + # Use a filler that puts the title at ~55 000 chars + filler_line = '<meta name="description" content="' + "B" * 200 + '"/>\n' + filler_count = 230 # ~235 bytes * 230 ≈ 54 050 chars before <title> + head_junk = filler_line * filler_count + page = ( + f"<html><head>{head_junk}" + f"<title>ASIN B0B9CGQ14V - Echo Dot (5th Gen)" + f"body content" + ).encode("utf-8") + title_pos = page.find(b" at {title_pos}, expected > 8192") + result = extract_title(page) + self.assertEqual(result, "ASIN B0B9CGQ14V - Echo Dot (5th Gen)") + + # ------------------------------------------------------------------ + # Baseline: small pages must continue to work + # ------------------------------------------------------------------ + + def test_normal_small_page(self): + """Standard small page should extract title correctly.""" + page = b"Simple Pagetext" + self.assertEqual(extract_title(page), "Simple Page") + + def test_str_input_small_page(self): + """str input small page.""" + page = "String Input" + self.assertEqual(extract_title(page), "String Input") + + # ------------------------------------------------------------------ + # Edge cases + # ------------------------------------------------------------------ + + def test_no_title_tag_returns_none(self): + """No in document → None.""" + page = b"<html><head></head><body>no title here</body></html>" + self.assertIsNone(extract_title(page)) + + def test_empty_bytes_returns_none(self): + """Empty bytes → None.""" + self.assertIsNone(extract_title(b"")) + + def test_html_entities_decoded(self): + """HTML entities inside <title> must be decoded.""" + page = b"<html><head><title>Café & Tea" + self.assertEqual(extract_title(page), "Café & Tea") + + def test_extra_whitespace_collapsed(self): + """Leading/trailing/internal whitespace in title is collapsed.""" + page = b" Multiple Spaces " + self.assertEqual(extract_title(page), "Multiple Spaces") + + def test_title_with_attributes_on_tag(self): + """ (tag with attributes) must still match.""" + page = b'<html><head><title lang="en">Attributed Title' + self.assertEqual(extract_title(page), "Attributed Title") + + def test_long_title_capped_at_2000_chars(self): + """Titles longer than 2 000 chars are capped.""" + long_title = "T" * 3000 + page = f"{long_title}".encode() + result = extract_title(page) + self.assertIsNotNone(result) + self.assertEqual(len(result), 2000) + + def test_title_300_chars_preserved(self): + """Titles up to 2 000 chars are preserved in full.""" + title = "X" * 300 + page = f"{title}".encode() + self.assertEqual(extract_title(page), title) + + def test_unsupported_type_returns_none(self): + """Passing an unsupported type (e.g. int) returns None without raising.""" + self.assertIsNone(extract_title(12345)) # type: ignore[arg-type] + + +if __name__ == "__main__": + unittest.main()