diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index e3b34f49c..7eade3bc6 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -95,7 +95,13 @@ TEXT_FILTER_LIST_LINE_SUFFIX = "
" TRANSLATE_WHITESPACE_TABLE = str.maketrans('', '', '\r\n\t ') PERL_STYLE_REGEX = r'^/(.*?)/([a-z]*)?$' -TITLE_RE = re.compile(r"]*>(.*?)", re.I | re.S) +# Whitespace is allowed after the tag name ("") but not after "<" - "< title>" is text, not a tag +TITLE_RE = re.compile(r"]*>(.*?)", re.I | re.S) +# Locate the tag with pos/endpos instead of data.lower(), which copies the whole document (67MB seen +# for one page) and, for str, can change its length ('İ' lowers to 2 chars) so the offset is wrong. +TITLE_TAG_STR_RE = re.compile(r"]+charset=["\']?\s*([a-z0-9_\-:+.]+)', re.I) # jq builtins that can leak sensitive data or cause harm when user-supplied expressions are executed. @@ -918,14 +924,18 @@ def extract_title(data: bytes | str, sniff_bytes: int = 2048, scan_chars: int = # rare but possible. We read up to 128 KiB from the tag onwards to handle # even pathological cases without scanning the whole document. _TITLE_WINDOW = 131072 + # Only look for the tag in the first 1 MiB (chars, or bytes for 8-bit). Well past any real , + # and a huge page is never scanned end to end. + _TITLE_SEARCH_LIMIT = 1024 * 1024 try: match data: case bytes() if data.startswith((b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff")): # UTF-32: locate the tag in the raw bytes, then decode the window. - tag_pos = data.lower().find(b"<\x00\x00\x00t\x00\x00\x00") - if tag_pos == -1: + tag_m = TITLE_TAG_UTF32_RE.search(data, 0, _TITLE_SEARCH_LIMIT * 4) + if not tag_m: return None + tag_pos = tag_m.start() chunk = data[tag_pos: tag_pos + _TITLE_WINDOW * 4].decode("utf-32", errors="replace") prefix = chunk case bytes() if data.startswith((b"\xff\xfe", b"\xfe\xff")): @@ -934,9 +944,10 @@ def extract_title(data: bytes | str, sniff_bytes: int = 2048, scan_chars: int = prefix = data[: max(scan_chars * 2, _TITLE_WINDOW)].decode("utf-16", errors="replace") case bytes(): # UTF-8 / legacy 8-bit: find the tag cheaply in raw bytes. - tag_pos = data.lower().find(b"İstanbul" + self.assertEqual(extract_title(page), "İstanbul") + self.assertEqual(extract_title(page.encode("utf-8")), "İstanbul") + + def test_uppercase_title_tag(self): + """Tag matching stays case-insensitive for str and bytes.""" + page = "Shouty" + self.assertEqual(extract_title(page), "Shouty") + self.assertEqual(extract_title(page.encode()), "Shouty") + + def test_whitespace_in_tags(self): + """Whitespace after the tag name is valid HTML, after '<' it is text (matches browsers/lxml).""" + self.assertEqual(extract_title("A"), "A") + self.assertEqual(extract_title("B"), "B") + self.assertEqual(extract_title("C"), "C") + self.assertEqual(extract_title(b"D</title\n></head>"), "D") + self.assertIsNone(extract_title("<head>< title>E")) + + def test_utf32_title(self): + page = "Wide".encode("utf-32") + self.assertEqual(extract_title(page), "Wide") + + def test_title_past_search_limit_is_ignored(self): + """A beyond the first 1 MiB is not searched for.""" + filler = "<!-- " + "A" * (1024 * 1024) + " -->" + page = f"<html><head>{filler}<title>Too far" + self.assertIsNone(extract_title(page)) + self.assertIsNone(extract_title(page.encode())) + if __name__ == "__main__": unittest.main()