fix(detect): normalize Unicode so accented ignore patterns match on macOS

An ignore rule naming a directory with an accent silently does nothing on
macOS, and the files it was meant to exclude get scanned anyway.

macOS (APFS/HFS+) returns filenames in NFD — "ç" comes back as "c" + U+0327
COMBINING CEDILLA — while editors write ignore files in NFC, where the same
"ç" is the single codepoint U+00E7. The two render identically and compare
unequal, so `fnmatch` never matches and the pattern is a no-op.

Found in a real repo: a `.graphifyignore` containing `Orçamento/` failed to
exclude that directory, and 9 client contract PDFs were picked up for semantic
extraction — i.e. queued to be sent to an LLM — despite an explicit rule
against it. The failure is silent: there is no warning, and the only symptom
is a file count that does not match what you expect. A user who does not
count would never know. That is what makes this worth fixing rather than
documenting: the rule appears to work.

Both sides are now normalized to NFC before matching, at three boundaries:
the pattern (in `_parse_gitignore_line`, so it covers .graphifyignore,
.gitignore and $GIT_DIR/info/exclude alike) and the two path forms used in
`_is_ignored` (`target.name` and the anchor-relative path).

NFC is already the form Linux and Windows produce, so this is a no-op there
and only repairs the macOS mismatch.

Tests: two regression tests cover both directions (NFC pattern vs NFD path on
disk, and the reverse); both fail before this change and pass after. A third
asserts ASCII patterns are unaffected, so the normalization cannot regress
existing behavior.

Full suite: 3833 passed. The 13 failures in tests/test_terraform.py are
pre-existing on a clean upstream checkout (optional tree_sitter_hcl not
installed) and unrelated to this change.
This commit is contained in:
Bruno Santanna
2026-08-11 15:18:33 +01:00
committed by safishamsi
parent 1fdd11fa76
commit 5ffaaa606a
2 changed files with 76 additions and 3 deletions
+21 -3
View File
@@ -5,6 +5,7 @@ import json
import os
import re
import shlex
import unicodedata
from concurrent.futures import ThreadPoolExecutor
from enum import Enum
from functools import lru_cache
@@ -928,6 +929,23 @@ def _is_noise_dir(part: str, parent: "Path | None" = None) -> bool:
_VCS_MARKERS = (".git", ".hg", ".svn", "_darcs", ".fossil")
def _nfc(text: str) -> str:
"""Normalize text to NFC so ignore matching survives Unicode form drift.
macOS (APFS/HFS+) returns filenames in NFD: "ç" comes back as "c" +
U+0327 COMBINING CEDILLA. Editors write ignore files in NFC, where the
same "ç" is the single codepoint U+00E7. The two render identically and
compare unequal, so a pattern like `Orçamento/` silently fails to exclude
the directory it names — the files are scanned and, for docs/PDFs, sent
to an LLM despite an explicit rule against it.
Both sides are normalized to NFC before any fnmatch call. NFC is the form
Linux and Windows already use, so this is a no-op there and only repairs
the macOS mismatch.
"""
return unicodedata.normalize("NFC", text)
def _parse_gitignore_line(raw: str) -> str:
"""Parse one raw line from a .graphifyignore file per gitignore spec.
@@ -949,7 +967,7 @@ def _parse_gitignore_line(raw: str) -> str:
line = line.replace("\\#", "#")
# Remove unescaped trailing spaces (per gitignore spec)
line = re.sub(r"(?<!\\) +$", "", line)
return line
return _nfc(line)
def _find_vcs_root(start: Path) -> Path | None:
@@ -1140,7 +1158,7 @@ def _is_ignored(
parts = rel.split("/")
if fnmatch.fnmatch(rel, p):
return True
if fnmatch.fnmatch(target.name, p):
if fnmatch.fnmatch(_nfc(target.name), p):
return True
for i, part in enumerate(parts):
if fnmatch.fnmatch(part, p):
@@ -1166,7 +1184,7 @@ def _is_ignored(
# ignore file governs its directory's contents, not the directory.
matched = False
try:
rel_anchor = str(target.relative_to(anchor)).replace(os.sep, "/")
rel_anchor = _nfc(str(target.relative_to(anchor)).replace(os.sep, "/"))
except ValueError:
continue # target outside this pattern's anchor: cannot match
if rel_anchor != ".":
+55
View File
@@ -119,6 +119,61 @@ def test_graphifyignore_excludes_file(tmp_path):
assert result["graphifyignore_patterns"] == 2
def test_graphifyignore_matches_nfd_path_with_nfc_pattern(tmp_path):
"""An accented pattern excludes its directory even when the FS stores NFD.
macOS returns filenames in NFD ("c" + U+0327) while editors write ignore
files in NFC (U+00E7). Without normalization the two compare unequal and
the rule silently does nothing — the files get scanned, and docs/PDFs are
sent to an LLM despite an explicit exclusion.
"""
nfc_name = unicodedata.normalize("NFC", "Or\u00e7amento")
nfd_name = unicodedata.normalize("NFD", nfc_name)
assert nfc_name != nfd_name # guard: the two forms really do differ
(tmp_path / ".graphifyignore").write_text(f"{nfc_name}/\n")
secret_dir = tmp_path / nfd_name
secret_dir.mkdir()
(secret_dir / "contrato.py").write_text("x = 1")
(tmp_path / "main.py").write_text("print('hi')")
result = detect(tmp_path)
file_list = result["files"]["code"]
assert any("main.py" in f for f in file_list)
assert not any("contrato.py" in f for f in file_list)
def test_graphifyignore_matches_nfc_path_with_nfd_pattern(tmp_path):
"""The reverse direction also holds: NFD pattern, NFC path on disk."""
nfc_name = unicodedata.normalize("NFC", "Or\u00e7amento")
nfd_name = unicodedata.normalize("NFD", nfc_name)
(tmp_path / ".graphifyignore").write_text(f"{nfd_name}/\n")
d = tmp_path / nfc_name
d.mkdir()
(d / "contrato.py").write_text("x = 1")
(tmp_path / "main.py").write_text("print('hi')")
result = detect(tmp_path)
file_list = result["files"]["code"]
assert any("main.py" in f for f in file_list)
assert not any("contrato.py" in f for f in file_list)
def test_graphifyignore_ascii_patterns_unaffected(tmp_path):
"""Normalization is a no-op for ASCII patterns — no regression."""
(tmp_path / ".graphifyignore").write_text("vendor/\n")
v = tmp_path / "vendor"
v.mkdir()
(v / "lib.py").write_text("x = 1")
(tmp_path / "main.py").write_text("x = 1")
result = detect(tmp_path)
file_list = result["files"]["code"]
assert any("main.py" in f for f in file_list)
assert not any("vendor" in f for f in file_list)
def test_graphifyignore_missing_is_fine(tmp_path):
"""No .graphifyignore is not an error."""
(tmp_path / "main.py").write_text("x = 1")