From b31dd1472c091891ee6a87cb46e8313cf3dde9be Mon Sep 17 00:00:00 2001 From: squidfunk Date: Fri, 15 May 2026 10:41:12 +0200 Subject: [PATCH 1/3] fix: `$` at end of line breaks link validation (#659) fix: `$` at end of line breaks link validation Signed-off-by: squidfunk --- python/tests/unit/collectors/test_references.py | 5 +++++ python/zensical/collectors/references/cursor.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/python/tests/unit/collectors/test_references.py b/python/tests/unit/collectors/test_references.py index 9ea5c83..dc66d97 100644 --- a/python/tests/unit/collectors/test_references.py +++ b/python/tests/unit/collectors/test_references.py @@ -1672,6 +1672,11 @@ class TestMath: assert text(md, links[0].text) == b"before" assert text(md, links[0].href) == b"href" + def test_no_math(self) -> None: + md = b"1.000$" + refs = collect(md) + assert len(refs) == 0 + # --------------------------------------------------------------------------- diff --git a/python/zensical/collectors/references/cursor.py b/python/zensical/collectors/references/cursor.py index b79f2cc..2240c33 100644 --- a/python/zensical/collectors/references/cursor.py +++ b/python/zensical/collectors/references/cursor.py @@ -1021,12 +1021,12 @@ def _scan_inline_code(cursor: Cursor) -> int | None: def _scan_math_inline(cursor: Cursor) -> int | None: """Scan for `$...$` math.""" end = cursor.pos - if end >= cursor.end and cursor.data[end] != _DOLLAR: + if end >= cursor.end or cursor.data[end] != _DOLLAR: return None # Skip opening $ and abort if it's followed by whitespace or another $ end += 1 - if end >= cursor.end and ( + if end >= cursor.end or ( cursor.data[end] in (_DOLLAR, _SPACE, _TAB, _NL, _CR) ): return None From a418c6bd3c62673e91df01690fd9eccafe262041 Mon Sep 17 00:00:00 2001 From: squidfunk Date: Fri, 15 May 2026 11:05:07 +0200 Subject: [PATCH 2/3] fix: link validation doesn't ignore fenced code blocks when `\r` is present Signed-off-by: squidfunk --- python/tests/unit/collectors/test_references.py | 8 ++++++++ python/zensical/collectors/references/cursor.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/python/tests/unit/collectors/test_references.py b/python/tests/unit/collectors/test_references.py index dc66d97..9b38013 100644 --- a/python/tests/unit/collectors/test_references.py +++ b/python/tests/unit/collectors/test_references.py @@ -1470,6 +1470,14 @@ class TestFencedCodeBlocks: b"~~~~ py\n[text](href)\n~~~~", id="fenced-code-4-tildes-lang", ), + pytest.param( + b"\n```\n[Start][]\n```\n", + id="fenced-code-with-info", + ), + pytest.param( + b"```\r\n[Start]\r\n```\r\n", + id="fenced-code-crlf-with-shortcut-link-ref", + ), ], ) def test_fenced_code(self, md: bytes) -> None: diff --git a/python/zensical/collectors/references/cursor.py b/python/zensical/collectors/references/cursor.py index 2240c33..6473405 100644 --- a/python/zensical/collectors/references/cursor.py +++ b/python/zensical/collectors/references/cursor.py @@ -971,7 +971,7 @@ def _scan_fenced_code(cursor: Cursor, char: int) -> int | None: # Skip to next line and terminate if we found a closing fence if end - start == opening: rest = _skip_whitespace(cursor, end) - if rest >= cursor.end or cursor.data[rest] == _NL: + if rest >= cursor.end or cursor.data[rest] in (_CR, _NL): return _skip_line(cursor, end) # Skip to next line From 4355dad3ce47a0e647e2187a770a2b9b0b2d502e Mon Sep 17 00:00:00 2001 From: squidfunk Date: Fri, 15 May 2026 11:23:45 +0200 Subject: [PATCH 3/3] fix: harden link validation for files with CRLF line endings Signed-off-by: squidfunk --- .../tests/unit/collectors/test_references.py | 145 +++++++++++++++++- .../zensical/collectors/references/cursor.py | 95 ++++++++---- 2 files changed, 204 insertions(+), 36 deletions(-) diff --git a/python/tests/unit/collectors/test_references.py b/python/tests/unit/collectors/test_references.py index 9b38013..78b93f7 100644 --- a/python/tests/unit/collectors/test_references.py +++ b/python/tests/unit/collectors/test_references.py @@ -569,8 +569,15 @@ class TestLinkDefinitions: assert text(md, link_defs[0].id) == expected_id assert text(md, link_defs[0].href) == expected_href - def test_link_def_title_on_next_line(self) -> None: - md = b'[id]: href\n "Title"' + @pytest.mark.parametrize( + "md", + [ + pytest.param(b'[id]: href\n "Title"', id="lf"), + pytest.param(b'[id]: href\r\n "Title"', id="crlf"), + pytest.param(b'[id]: href\r "Title"', id="cr"), + ], + ) + def test_link_def_title_on_next_line(self, md: bytes) -> None: refs = collect(md) assert len(refs) == 1 @@ -579,6 +586,34 @@ class TestLinkDefinitions: assert text(md, link_defs[0].id) == b"id" assert text(md, link_defs[0].href) == b"href" + @pytest.mark.parametrize( + "md", + [ + pytest.param( + b"[id]: \r\n[after](href)", + id="crlf", + ), + pytest.param( + b"[id]: \r[after](href)", + id="cr", + ), + ], + ) + def test_link_def_angle_brackets_with_link_after( + self, md: bytes + ) -> None: + refs = collect(md) + assert len(refs) == 2 + + link_defs = link_defs_only(refs) + assert len(link_defs) == 1 + assert text(md, link_defs[0].id) == b"id" + assert text(md, link_defs[0].href) == b"href" + + links = links_only(refs) + assert len(links) == 1 + assert text(md, links[0].text) == b"after" + # --- negative cases --- def test_no_link_def_empty_href(self) -> None: @@ -1205,6 +1240,18 @@ class TestWikilinks: b"id", id="link-ref-with-newline", ), + pytest.param( + b"[[Page]]\r\n[id]", + b"[Page]", + b"id", + id="link-ref-with-crlf", + ), + pytest.param( + b"[[Page]]\r[id]", + b"[Page]", + b"id", + id="link-ref-with-cr", + ), ], ) def test_no_wikilink( @@ -1478,6 +1525,10 @@ class TestFencedCodeBlocks: b"```\r\n[Start]\r\n```\r\n", id="fenced-code-crlf-with-shortcut-link-ref", ), + pytest.param( + b"```\r[Start]\r```\r", + id="fenced-code-cr-with-shortcut-link-ref", + ), ], ) def test_fenced_code(self, md: bytes) -> None: @@ -1646,6 +1697,14 @@ class TestMath: b"\\[\n[text](href)\n\\]", id="match-block-brackets", ), + pytest.param( + b"\\[\r\n[text](href)\r\n\\]", + id="match-block-brackets-crlf", + ), + pytest.param( + b"\\[\r[text](href)\r\\]", + id="match-block-brackets-cr", + ), pytest.param( b"\\[ [text](href) \\]", id="match-block-brackets, single line", @@ -1685,6 +1744,22 @@ class TestMath: refs = collect(md) assert len(refs) == 0 + @pytest.mark.parametrize( + "md", + [ + pytest.param(b"$[text](href)\n$", id="lf"), + pytest.param(b"$[text](href)\r\n$", id="crlf"), + pytest.param(b"$[text](href)\r$", id="cr"), + ], + ) + def test_no_math_inline_across_line_ending(self, md: bytes) -> None: + refs = collect(md) + assert len(refs) == 1 + + links = links_only(refs) + assert len(links) == 1 + assert text(md, links[0].text) == b"text" + # --------------------------------------------------------------------------- @@ -1705,6 +1780,16 @@ class TestHtmlLinks: b"href", id="html-a-href, in block", ), + pytest.param( + b'
\r\ntext\r\n
', + b"href", + id="html-a-href, in block, crlf", + ), + pytest.param( + b'
\rtext\r
', + b"href", + id="html-a-href, in block, cr", + ), pytest.param( b'', b"image.png", @@ -1911,6 +1996,14 @@ class TestJinja: b"{% if [text](href) %}...{% endif %}", id="jinja-block", ), + pytest.param( + b"{% if\r\n[text](href)\r\n%}", + id="jinja-block-crlf", + ), + pytest.param( + b"{% if\r[text](href)\r%}", + id="jinja-block-cr", + ), pytest.param( b"{{ [text](href) }}", id="jinja-expr", @@ -1925,6 +2018,22 @@ class TestJinja: refs = collect(md) assert len(refs) == 0 + @pytest.mark.parametrize( + "md", + [ + pytest.param(b"{% if\n\n[text](href) %}", id="lf"), + pytest.param(b"{% if\r\n\r\n[text](href) %}", id="crlf"), + pytest.param(b"{% if\r\r[text](href) %}", id="cr"), + ], + ) + def test_refs_after_jinja_blank_line(self, md: bytes) -> None: + refs = collect(md) + assert len(refs) == 1 + + links = links_only(refs) + assert len(links) == 1 + assert text(md, links[0].text) == b"text" + # --------------------------------------------------------------------------- @@ -1932,11 +2041,39 @@ class TestJinja: class TestExclusions: """Tests for further exclusions.""" - def test_abbreviations(self) -> None: - md = b"*[abbr]: text\n" + @pytest.mark.parametrize( + "md", + [ + pytest.param(b"*[abbr]: text\n", id="lf"), + pytest.param(b"*[abbr]: text\r\n", id="crlf"), + pytest.param(b"*[abbr]: text\r", id="cr"), + ], + ) + def test_abbreviations(self, md: bytes) -> None: refs = collect(md) assert len(refs) == 0 + @pytest.mark.parametrize( + "md", + [ + pytest.param( + b"*[abbr]: text\r\n[after](href)", + id="crlf", + ), + pytest.param( + b"*[abbr]: text\r[after](href)", + id="cr", + ), + ], + ) + def test_abbreviation_with_link_after(self, md: bytes) -> None: + refs = collect(md) + assert len(refs) == 1 + + links = links_only(refs) + assert len(links) == 1 + assert text(md, links[0].text) == b"after" + @pytest.mark.parametrize( "md", [ diff --git a/python/zensical/collectors/references/cursor.py b/python/zensical/collectors/references/cursor.py index 6473405..a6e5c39 100644 --- a/python/zensical/collectors/references/cursor.py +++ b/python/zensical/collectors/references/cursor.py @@ -139,7 +139,7 @@ class Cursor: """Advance the cursor by `n` bytes.""" for _ in range(n): if self.pos < self.end: - if self.data[self.pos] == _NL: + if self.data[self.pos] in (_CR, _NL): self.col = 0 else: self.col += 1 @@ -152,7 +152,7 @@ class Cursor: def _is_blank_prefix(self) -> bool: """Return whether there's only whitespace before the cursor.""" i = self.pos - 1 - while i >= 0 and self.data[i] != _NL: + while i >= 0 and self.data[i] not in (_CR, _NL): if self.data[i] not in _WHITESPACE: return False i -= 1 @@ -206,7 +206,7 @@ def _scan(cursor: Cursor) -> Iterator[Reference]: if ( cursor.at_line_start() and count >= 3 # noqa: PLR2004 - and cursor.peek(count) in (_SPACE, _TAB, _NL, _CR, -1) + and cursor.peek(count) in (_SPACE, _TAB, _CR, _NL, -1) ): end = _scan_fenced_code(cursor, _BACKTICK) if end is not None: @@ -462,7 +462,7 @@ def _scan_link_def(cursor: Cursor) -> LinkDefinition | None: result = _scan_link_href_in_angle_brackets(cursor, end) if result is not None: href, end = result - while end < cursor.end and cursor.data[end] != _NL: + while end < cursor.end and cursor.data[end] not in (_CR, _NL): end += 1 # Advance cursor and return link definition @@ -477,7 +477,7 @@ def _scan_link_def(cursor: Cursor) -> LinkDefinition | None: # Consume link href until whitespace or quote begin = end while end < cursor.end: - if cursor.data[end] in (_SPACE, _TAB, _NL, _CR, _QUOTE): + if cursor.data[end] in (_SPACE, _TAB, _CR, _NL, _QUOTE): break # Literal @@ -557,7 +557,7 @@ def _scan_link_href(cursor: Cursor, start: int) -> tuple[Span, int] | None: pos = _skip_whitespace_newline(cursor, pos) if pos < cursor.end and cursor.data[pos] == _QUOTE: while pos < cursor.end and cursor.data[pos] != _RPAREN: - if cursor.data[pos] == _NL: + if cursor.data[pos] in (_CR, _NL): return None # Literal @@ -599,7 +599,7 @@ def _scan_link_href(cursor: Cursor, start: int) -> tuple[Span, int] | None: # Skip optional title and whitespace href = cursor.span(start + 1, end) while end < cursor.end and cursor.data[end] != _RPAREN: - if cursor.data[end] == _NL: + if cursor.data[end] in (_CR, _NL): return None # Literal @@ -624,7 +624,7 @@ def _scan_link_href_in_angle_brackets( while ( end < cursor.end and cursor.data[end] != _RANGLE - and cursor.data[end] != _NL + and cursor.data[end] not in (_CR, _NL) ): end += 1 @@ -911,7 +911,7 @@ def _scan_footnote_def( # Trim trailing newlines from the body span body_end = end - while body_end > body_start and cursor.data[body_end - 1] in (_NL, _CR): + while body_end > body_start and cursor.data[body_end - 1] in (_CR, _NL): body_end -= 1 # Build and emit the definition @@ -1027,12 +1027,12 @@ def _scan_math_inline(cursor: Cursor) -> int | None: # Skip opening $ and abort if it's followed by whitespace or another $ end += 1 if end >= cursor.end or ( - cursor.data[end] in (_DOLLAR, _SPACE, _TAB, _NL, _CR) + cursor.data[end] in (_DOLLAR, _SPACE, _TAB, _CR, _NL) ): return None # Scan for closing $ - while end < cursor.end and cursor.data[end] != _NL: + while end < cursor.end and cursor.data[end] not in (_CR, _NL): if ( cursor.data[end] == _DOLLAR # Closing $ must not be preceded by whitespace or $ @@ -1075,7 +1075,7 @@ def _scan_math( # Consume opening delimiter and scan for closing delimiter start += 2 while start < cursor.end: - if not multiline and cursor.data[start] == _NL: + if not multiline and cursor.data[start] in (_CR, _NL): return None # Consume closing delimiter and return if we found it @@ -1147,7 +1147,7 @@ def _scan_html_attrs( """Scans for HTML attributes.""" attrs: dict[bytes, Span] = {} while pos < cursor.end and cursor.data[pos] != _RANGLE: - if cursor.data[pos] in (_SPACE, _TAB, _NL, _CR, _SLASH): + if cursor.data[pos] in (_SPACE, _TAB, _CR, _NL, _SLASH): pos += 1 continue @@ -1163,7 +1163,7 @@ def _scan_html_attrs( start = pos while pos < cursor.end and ( cursor.data[pos] - not in (_SPACE, _TAB, _NL, _CR, _RANGLE, _SLASH, _EQUALS) + not in (_SPACE, _TAB, _CR, _NL, _RANGLE, _SLASH, _EQUALS) ): pos += 1 @@ -1171,7 +1171,7 @@ def _scan_html_attrs( name = cursor.data[start:pos].lower() # Skip whitespace before = - while pos < cursor.end and cursor.data[pos] in (_SPACE, _TAB, _NL, _CR): + while pos < cursor.end and cursor.data[pos] in (_SPACE, _TAB, _CR, _NL): pos += 1 # Boolean attribute - record presence with empty span @@ -1181,7 +1181,7 @@ def _scan_html_attrs( # Skip = and whitespace after it pos += 1 - while pos < cursor.end and cursor.data[pos] in (_SPACE, _TAB, _NL, _CR): + while pos < cursor.end and cursor.data[pos] in (_SPACE, _TAB, _CR, _NL): pos += 1 # Quoted value @@ -1203,7 +1203,7 @@ def _scan_html_attrs( else: start = pos while pos < cursor.end and ( - cursor.data[pos] not in (_SPACE, _TAB, _NL, _CR, _RANGLE) + cursor.data[pos] not in (_SPACE, _TAB, _CR, _NL, _RANGLE) ): pos += 1 @@ -1240,7 +1240,7 @@ def _scan_html_tag(cursor: Cursor) -> tuple[int, list[Link]] | None: # Must be followed by whitespace, >, or / if end >= cursor.end or ( - cursor.data[end] not in (_SPACE, _TAB, _NL, _CR, _RANGLE, _SLASH) + cursor.data[end] not in (_SPACE, _TAB, _CR, _NL, _RANGLE, _SLASH) ): return None @@ -1289,7 +1289,7 @@ def _scan_html_block(cursor: Cursor) -> int | None: # Skip if there are newlines in the opening tag pos, attrs = result - if b"\n" in cursor.data[start:pos]: + if b"\n" in cursor.data[start:pos] or b"\r" in cursor.data[start:pos]: return None # If markdown attribute is set to a valid value, the md_in_html extension @@ -1302,9 +1302,7 @@ def _scan_html_block(cursor: Cursor) -> int | None: # Inline form: opening and closing tag on the same line closing_tag = b"" - eol = cursor.data.find(b"\n", pos) - if eol == -1: - eol = cursor.end + eol = _find_line_end(cursor, pos) # Search for closing tag on the same line close = cursor.data.find(closing_tag, pos, eol + 1) @@ -1313,7 +1311,7 @@ def _scan_html_block(cursor: Cursor) -> int | None: # Multi-line form: require a newline (with optional trailing whitespace) i = _skip_whitespace(cursor, pos) - if i >= cursor.end or cursor.data[i] not in (_NL, _CR): + if i >= cursor.end or cursor.data[i] not in (_CR, _NL): return None if ( cursor.data[i] == _CR @@ -1331,8 +1329,7 @@ def _scan_html_block(cursor: Cursor) -> int | None: return None # Search for closing tag - start = cursor.data.rfind(b"\n", 0, pos) - start = 0 if start == -1 else start + 1 + start = _find_line_start(cursor, pos) if cursor.data[start:pos].strip() == b"": return _skip_line_ending(cursor, pos + len(closing_tag)) @@ -1360,19 +1357,25 @@ def _scan_jinja(cursor: Cursor) -> int | None: else: return None - # Scan for closing delimiter, allowing newlines but not blank lines + # Scan for closing delimiter, allowing line endings but not blank lines end = start + 2 newlines = 0 while end < cursor.end: char = cursor.data[end] - # Check for two consecutive newlines - if char == _NL: + # Check for two consecutive line endings + if char in (_CR, _NL): newlines += 1 - # Terminte if we encounter a blank line + # Terminate if we encounter a blank line if newlines == 2: # noqa: PLR2004 return None + if ( + char == _CR + and end + 1 < cursor.end + and cursor.data[end + 1] == _NL + ): + end += 1 else: newlines = 0 @@ -1451,7 +1454,13 @@ def _skip_whitespace(cursor: Cursor, pos: int) -> int: def _skip_whitespace_newline(cursor: Cursor, pos: int) -> int: """Skip trailing horizontal whitespace and at most one newline.""" pos = _skip_whitespace(cursor, pos) - if pos < cursor.end and cursor.data[pos] == _NL: + if pos < cursor.end and cursor.data[pos] in (_CR, _NL): + if ( + cursor.data[pos] == _CR + and pos + 1 < cursor.end + and cursor.data[pos + 1] == _NL + ): + pos += 1 pos = _skip_whitespace(cursor, pos + 1) return pos @@ -1469,7 +1478,14 @@ def _skip_line_ending(cursor: Cursor, pos: int) -> int: def _skip_line(cursor: Cursor, pos: int) -> int: """Skip to the next line, consuming the newline.""" - while pos < cursor.end and cursor.data[pos] != _NL: + while pos < cursor.end and cursor.data[pos] not in (_CR, _NL): + pos += 1 + if ( + pos < cursor.end + and cursor.data[pos] == _CR + and pos + 1 < cursor.end + and cursor.data[pos + 1] == _NL + ): pos += 1 if pos < cursor.end: pos += 1 @@ -1481,12 +1497,27 @@ def _find_bracket(cursor: Cursor, pos: int) -> int: while ( pos < cursor.end and cursor.data[pos] != _RBRACKET - and cursor.data[pos] != _NL + and cursor.data[pos] not in (_CR, _NL) ): pos += 1 return pos +def _find_line_end(cursor: Cursor, pos: int) -> int: + """Find the next line ending or end of buffer.""" + while pos < cursor.end and cursor.data[pos] not in (_CR, _NL): + pos += 1 + return pos + + +def _find_line_start(cursor: Cursor, pos: int) -> int: + """Find the first byte after the previous line ending.""" + start = pos - 1 + while start >= 0 and cursor.data[start] not in (_CR, _NL): + start -= 1 + return start + 1 + + # ----------------------------------------------------------------------------