fix: extract <title> from pages with large <head> sections (#4217) (#4220)

Co-authored-by: Charles Rossi <charles@choreless.dev>
2026-09-27 07:46:22 +00:00 · 2026-06-15 13:52:28 +02:00
parent 6f4cc2d6c1
commit be4718f2b8
2 changed files with 173 additions and 9 deletions
@@ -757,16 +757,41 @@ def get_triggered_text(content, trigger_text):


 def extract_title(data: bytes | str, sniff_bytes: int = 2048, scan_chars: int = 8192) -> str | None:
+    """Extract the <title> from an HTML document.
+
+    Rather than decoding/scanning a fixed prefix of the whole document, we first
+    locate the raw ``<title`` marker and then decode only a small window around
+    it.  This handles pages (e.g. Amazon) where large ``<head>`` sections push
+    the title tag well past the old 8 192-character scan limit.
+    """
+    # Maximum bytes/chars to extract after (and including) the opening <title tag.
+    # The regex needs to see </title>, so the window must cover the full content.
+    # The return value is always capped at 2 000 chars; titles beyond that are
+    # rare but possible.  We read up to 128 KiB from the tag onwards to handle
+    # even pathological cases without scanning the whole document.
+    _TITLE_WINDOW = 131072
+
    try:
-        # Only decode/process the prefix we need for title extraction
        match data:
-            case bytes() if data.startswith((b"\xff\xfe", b"\xfe\xff")):
-                prefix = data[:scan_chars * 2].decode("utf-16", errors="replace")
            case bytes() if data.startswith((b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff")):
-                prefix = data[:scan_chars * 4].decode("utf-32", errors="replace")
+                # UTF-32: locate the tag in the raw bytes, then decode the window.
+                tag_pos = data.lower().find(b"<\x00\x00\x00t\x00\x00\x00")
+                if tag_pos == -1:
+                    return None
+                chunk = data[tag_pos: tag_pos + _TITLE_WINDOW * 4].decode("utf-32", errors="replace")
+                prefix = chunk
+            case bytes() if data.startswith((b"\xff\xfe", b"\xfe\xff")):
+                # UTF-16: simple byte-pair search is tricky; fall back to decoding
+                # a reasonable head chunk and let the regex do the rest.
+                prefix = data[: max(scan_chars * 2, _TITLE_WINDOW)].decode("utf-16", errors="replace")
            case bytes():
+                # UTF-8 / legacy 8-bit: find the tag cheaply in raw bytes.
+                tag_pos = data.lower().find(b"<title")
+                if tag_pos == -1:
+                    return None
+                raw_chunk = data[tag_pos: tag_pos + _TITLE_WINDOW]
                try:
-                    prefix = data[:scan_chars].decode("utf-8")
+                    chunk = raw_chunk.decode("utf-8")
                except UnicodeDecodeError:
                    try:
                        head = data[:sniff_bytes].decode("ascii", errors="ignore")
@@ -774,23 +799,27 @@ def extract_title(data: bytes | str, sniff_bytes: int = 2048, scan_chars: int =
                            enc = m.group(1).lower()
                        else:
                            enc = "cp1252"
-                        prefix = data[:scan_chars * 2].decode(enc, errors="replace")
+                        chunk = raw_chunk.decode(enc, errors="replace")
                    except Exception as e:
                        logger.error(f"Title extraction encoding detection failed: {e}")
                        return None
+                prefix = chunk
            case str():
-                prefix = data[:scan_chars] if len(data) > scan_chars else data
+                tag_pos = data.lower().find("<title")
+                if tag_pos == -1:
+                    return None
+                prefix = data[tag_pos: tag_pos + _TITLE_WINDOW]
            case _:
                logger.error(f"Title extraction received unsupported data type: {type(data)}")
                return None

-        # Search only in the prefix
+        # Search only in the (now tag-anchored) prefix
        if m := TITLE_RE.search(prefix):
            title = html.unescape(" ".join(m.group(1).split())).strip()
            # Some safe limit
            return title[:2000]
        return None
-        
+
    except Exception as e:
        logger.error(f"Title extraction failed: {e}")
        return None
@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+# coding=utf-8
+
+"""Unit tests for html_tools.extract_title — including regression for #4217.
+
+Issue #4217: extract_title silently returns None for pages where <title> is
+pushed past the hard-coded 8 192-character scan window by large <head> content
+(e.g. Amazon product pages where <title> can sit at character index 55 000+).
+"""
+
+import unittest
+
+from changedetectionio.html_tools import extract_title
+
+
+def _make_large_head_page(title: str, filler_count: int = 500) -> bytes:
+    """Build a synthetic HTML page whose <title> is pushed far past 8 192 chars.
+
+    Each filler line is ~126 bytes; 500 lines ≈ 63 000 bytes before <title>.
+    """
+    filler_line = '<meta name="x" content="' + "A" * 100 + '"/>\n'
+    head_junk = filler_line * filler_count
+    page = (
+        f"<html><head>{head_junk}"
+        f"<title>{title}</title>"
+        f"</head><body></body></html>"
+    )
+    return page.encode("utf-8")
+
+
+class TestExtractTitle(unittest.TestCase):
+    # ------------------------------------------------------------------
+    # Regression: issue #4217 — large <head> pushes <title> past scan limit
+    # ------------------------------------------------------------------
+
+    def test_large_head_bytes_title_extracted(self):
+        """<title> beyond 8 192 bytes must still be extracted (bytes input)."""
+        page = _make_large_head_page("Amazon Product Title - Real Title Here")
+        title_pos = page.find(b"<title")
+        self.assertGreater(
+            title_pos,
+            8192,
+            f"Precondition: <title> must be past 8 192 chars (actual: {title_pos})",
+        )
+        result = extract_title(page)
+        self.assertEqual(result, "Amazon Product Title - Real Title Here")
+
+    def test_large_head_str_title_extracted(self):
+        """<title> beyond 8 192 chars must still be extracted (str input)."""
+        page_bytes = _make_large_head_page("Large Head String Test")
+        page_str = page_bytes.decode("utf-8")
+        title_pos = page_str.find("<title")
+        self.assertGreater(title_pos, 8192)
+        result = extract_title(page_str)
+        self.assertEqual(result, "Large Head String Test")
+
+    def test_very_large_head_55000_chars(self):
+        """Simulate Amazon-like pages where <title> is at ~55 000 chars."""
+        # Use a filler that puts the title at ~55 000 chars
+        filler_line = '<meta name="description" content="' + "B" * 200 + '"/>\n'
+        filler_count = 230  # ~235 bytes * 230 ≈ 54 050 chars before <title>
+        head_junk = filler_line * filler_count
+        page = (
+            f"<html><head>{head_junk}"
+            f"<title>ASIN B0B9CGQ14V - Echo Dot (5th Gen)</title>"
+            f"</head><body>body content</body></html>"
+        ).encode("utf-8")
+        title_pos = page.find(b"<title")
+        self.assertGreater(title_pos, 8192, f"<title> at {title_pos}, expected > 8192")
+        result = extract_title(page)
+        self.assertEqual(result, "ASIN B0B9CGQ14V - Echo Dot (5th Gen)")
+
+    # ------------------------------------------------------------------
+    # Baseline: small pages must continue to work
+    # ------------------------------------------------------------------
+
+    def test_normal_small_page(self):
+        """Standard small page should extract title correctly."""
+        page = b"<html><head><title>Simple Page</title></head><body>text</body></html>"
+        self.assertEqual(extract_title(page), "Simple Page")
+
+    def test_str_input_small_page(self):
+        """str input small page."""
+        page = "<html><head><title>String Input</title></head><body></body></html>"
+        self.assertEqual(extract_title(page), "String Input")
+
+    # ------------------------------------------------------------------
+    # Edge cases
+    # ------------------------------------------------------------------
+
+    def test_no_title_tag_returns_none(self):
+        """No <title> in document → None."""
+        page = b"<html><head></head><body>no title here</body></html>"
+        self.assertIsNone(extract_title(page))
+
+    def test_empty_bytes_returns_none(self):
+        """Empty bytes → None."""
+        self.assertIsNone(extract_title(b""))
+
+    def test_html_entities_decoded(self):
+        """HTML entities inside <title> must be decoded."""
+        page = b"<html><head><title>Caf&eacute; &amp; Tea</title></head><body></body></html>"
+        self.assertEqual(extract_title(page), "Café & Tea")
+
+    def test_extra_whitespace_collapsed(self):
+        """Leading/trailing/internal whitespace in title is collapsed."""
+        page = b"<html><head><title>  Multiple   Spaces  </title></head><body></body></html>"
+        self.assertEqual(extract_title(page), "Multiple Spaces")
+
+    def test_title_with_attributes_on_tag(self):
+        """<title lang="en"> (tag with attributes) must still match."""
+        page = b'<html><head><title lang="en">Attributed Title</title></head><body></body></html>'
+        self.assertEqual(extract_title(page), "Attributed Title")
+
+    def test_long_title_capped_at_2000_chars(self):
+        """Titles longer than 2 000 chars are capped."""
+        long_title = "T" * 3000
+        page = f"<html><head><title>{long_title}</title></head><body></body></html>".encode()
+        result = extract_title(page)
+        self.assertIsNotNone(result)
+        self.assertEqual(len(result), 2000)
+
+    def test_title_300_chars_preserved(self):
+        """Titles up to 2 000 chars are preserved in full."""
+        title = "X" * 300
+        page = f"<html><head><title>{title}</title></head><body></body></html>".encode()
+        self.assertEqual(extract_title(page), title)
+
+    def test_unsupported_type_returns_none(self):
+        """Passing an unsupported type (e.g. int) returns None without raising."""
+        self.assertIsNone(extract_title(12345))  # type: ignore[arg-type]
+
+
+if __name__ == "__main__":
+    unittest.main()