mirror of
https://github.com/safishamsi/graphify.git
synced 2026-08-27 00:36:39 +00:00
fix(detect): normalize Unicode so accented ignore patterns match on macOS
An ignore rule naming a directory with an accent silently does nothing on macOS, and the files it was meant to exclude get scanned anyway. macOS (APFS/HFS+) returns filenames in NFD — "ç" comes back as "c" + U+0327 COMBINING CEDILLA — while editors write ignore files in NFC, where the same "ç" is the single codepoint U+00E7. The two render identically and compare unequal, so `fnmatch` never matches and the pattern is a no-op. Found in a real repo: a `.graphifyignore` containing `Orçamento/` failed to exclude that directory, and 9 client contract PDFs were picked up for semantic extraction — i.e. queued to be sent to an LLM — despite an explicit rule against it. The failure is silent: there is no warning, and the only symptom is a file count that does not match what you expect. A user who does not count would never know. That is what makes this worth fixing rather than documenting: the rule appears to work. Both sides are now normalized to NFC before matching, at three boundaries: the pattern (in `_parse_gitignore_line`, so it covers .graphifyignore, .gitignore and $GIT_DIR/info/exclude alike) and the two path forms used in `_is_ignored` (`target.name` and the anchor-relative path). NFC is already the form Linux and Windows produce, so this is a no-op there and only repairs the macOS mismatch. Tests: two regression tests cover both directions (NFC pattern vs NFD path on disk, and the reverse); both fail before this change and pass after. A third asserts ASCII patterns are unaffected, so the normalization cannot regress existing behavior. Full suite: 3833 passed. The 13 failures in tests/test_terraform.py are pre-existing on a clean upstream checkout (optional tree_sitter_hcl not installed) and unrelated to this change.
This commit is contained in:
committed by
safishamsi
parent
1fdd11fa76
commit
5ffaaa606a
+21
-3
@@ -5,6 +5,7 @@ import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import unicodedata
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from enum import Enum
|
||||
from functools import lru_cache
|
||||
@@ -928,6 +929,23 @@ def _is_noise_dir(part: str, parent: "Path | None" = None) -> bool:
|
||||
_VCS_MARKERS = (".git", ".hg", ".svn", "_darcs", ".fossil")
|
||||
|
||||
|
||||
def _nfc(text: str) -> str:
|
||||
"""Normalize text to NFC so ignore matching survives Unicode form drift.
|
||||
|
||||
macOS (APFS/HFS+) returns filenames in NFD: "ç" comes back as "c" +
|
||||
U+0327 COMBINING CEDILLA. Editors write ignore files in NFC, where the
|
||||
same "ç" is the single codepoint U+00E7. The two render identically and
|
||||
compare unequal, so a pattern like `Orçamento/` silently fails to exclude
|
||||
the directory it names — the files are scanned and, for docs/PDFs, sent
|
||||
to an LLM despite an explicit rule against it.
|
||||
|
||||
Both sides are normalized to NFC before any fnmatch call. NFC is the form
|
||||
Linux and Windows already use, so this is a no-op there and only repairs
|
||||
the macOS mismatch.
|
||||
"""
|
||||
return unicodedata.normalize("NFC", text)
|
||||
|
||||
|
||||
def _parse_gitignore_line(raw: str) -> str:
|
||||
"""Parse one raw line from a .graphifyignore file per gitignore spec.
|
||||
|
||||
@@ -949,7 +967,7 @@ def _parse_gitignore_line(raw: str) -> str:
|
||||
line = line.replace("\\#", "#")
|
||||
# Remove unescaped trailing spaces (per gitignore spec)
|
||||
line = re.sub(r"(?<!\\) +$", "", line)
|
||||
return line
|
||||
return _nfc(line)
|
||||
|
||||
|
||||
def _find_vcs_root(start: Path) -> Path | None:
|
||||
@@ -1140,7 +1158,7 @@ def _is_ignored(
|
||||
parts = rel.split("/")
|
||||
if fnmatch.fnmatch(rel, p):
|
||||
return True
|
||||
if fnmatch.fnmatch(target.name, p):
|
||||
if fnmatch.fnmatch(_nfc(target.name), p):
|
||||
return True
|
||||
for i, part in enumerate(parts):
|
||||
if fnmatch.fnmatch(part, p):
|
||||
@@ -1166,7 +1184,7 @@ def _is_ignored(
|
||||
# ignore file governs its directory's contents, not the directory.
|
||||
matched = False
|
||||
try:
|
||||
rel_anchor = str(target.relative_to(anchor)).replace(os.sep, "/")
|
||||
rel_anchor = _nfc(str(target.relative_to(anchor)).replace(os.sep, "/"))
|
||||
except ValueError:
|
||||
continue # target outside this pattern's anchor: cannot match
|
||||
if rel_anchor != ".":
|
||||
|
||||
@@ -119,6 +119,61 @@ def test_graphifyignore_excludes_file(tmp_path):
|
||||
assert result["graphifyignore_patterns"] == 2
|
||||
|
||||
|
||||
def test_graphifyignore_matches_nfd_path_with_nfc_pattern(tmp_path):
|
||||
"""An accented pattern excludes its directory even when the FS stores NFD.
|
||||
|
||||
macOS returns filenames in NFD ("c" + U+0327) while editors write ignore
|
||||
files in NFC (U+00E7). Without normalization the two compare unequal and
|
||||
the rule silently does nothing — the files get scanned, and docs/PDFs are
|
||||
sent to an LLM despite an explicit exclusion.
|
||||
"""
|
||||
nfc_name = unicodedata.normalize("NFC", "Or\u00e7amento")
|
||||
nfd_name = unicodedata.normalize("NFD", nfc_name)
|
||||
assert nfc_name != nfd_name # guard: the two forms really do differ
|
||||
|
||||
(tmp_path / ".graphifyignore").write_text(f"{nfc_name}/\n")
|
||||
secret_dir = tmp_path / nfd_name
|
||||
secret_dir.mkdir()
|
||||
(secret_dir / "contrato.py").write_text("x = 1")
|
||||
(tmp_path / "main.py").write_text("print('hi')")
|
||||
|
||||
result = detect(tmp_path)
|
||||
file_list = result["files"]["code"]
|
||||
assert any("main.py" in f for f in file_list)
|
||||
assert not any("contrato.py" in f for f in file_list)
|
||||
|
||||
|
||||
def test_graphifyignore_matches_nfc_path_with_nfd_pattern(tmp_path):
|
||||
"""The reverse direction also holds: NFD pattern, NFC path on disk."""
|
||||
nfc_name = unicodedata.normalize("NFC", "Or\u00e7amento")
|
||||
nfd_name = unicodedata.normalize("NFD", nfc_name)
|
||||
|
||||
(tmp_path / ".graphifyignore").write_text(f"{nfd_name}/\n")
|
||||
d = tmp_path / nfc_name
|
||||
d.mkdir()
|
||||
(d / "contrato.py").write_text("x = 1")
|
||||
(tmp_path / "main.py").write_text("print('hi')")
|
||||
|
||||
result = detect(tmp_path)
|
||||
file_list = result["files"]["code"]
|
||||
assert any("main.py" in f for f in file_list)
|
||||
assert not any("contrato.py" in f for f in file_list)
|
||||
|
||||
|
||||
def test_graphifyignore_ascii_patterns_unaffected(tmp_path):
|
||||
"""Normalization is a no-op for ASCII patterns — no regression."""
|
||||
(tmp_path / ".graphifyignore").write_text("vendor/\n")
|
||||
v = tmp_path / "vendor"
|
||||
v.mkdir()
|
||||
(v / "lib.py").write_text("x = 1")
|
||||
(tmp_path / "main.py").write_text("x = 1")
|
||||
|
||||
result = detect(tmp_path)
|
||||
file_list = result["files"]["code"]
|
||||
assert any("main.py" in f for f in file_list)
|
||||
assert not any("vendor" in f for f in file_list)
|
||||
|
||||
|
||||
def test_graphifyignore_missing_is_fine(tmp_path):
|
||||
"""No .graphifyignore is not an error."""
|
||||
(tmp_path / "main.py").write_text("x = 1")
|
||||
|
||||
Reference in New Issue
Block a user