diff --git a/CHANGELOG.md b/CHANGELOG.md index cb6599165..fc46faf6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.61 (unreleased) +- Fix: a prose note whose stem is bare plural "tokens" (`TOKENS.md`) is no longer dropped as a credential dump — it reads as a design-token reference doc, unlike singular `token.md` or another keyword's bare plural (`secrets.md`), which still exclude (#3527, thanks @conradkelly). - Fix: `graphify.serve` now imports cleanly on Python 3.12 and 3.13. The `chinese` extra pins `jieba-py` from 3.12 onward (0.9.60 mistakenly kept the old `jieba` until 3.14, and its invalid regex escapes are a hard error on 3.12+), and the jieba import now suppresses the tokenizer's `SyntaxWarning` regardless of message or line so it never escalates under `-W error`. - Fix: the git hook's rebuild-root guard now rejects a symlink-loop or dangling `.graphify_root` on Python 3.13, whose `Path.resolve()` no longer raises on a loop — the saved root must resolve to a real directory inside the repo before it is adopted. diff --git a/graphify/detect.py b/graphify/detect.py index 287b0f431..eec8aaf1d 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -183,6 +183,13 @@ _GENERIC_KEYWORD_PATTERNS = [ # stays excluded — see _is_prose_note. _PROSE_EXTS = frozenset({".md", ".markdown", ".rst", ".org", ".adoc", ".tex"}) +# Bare PLURAL "tokens" in a prose file (TOKENS.md) reads as a design-token +# reference doc, not a credential dump — unlike every other generic keyword's +# plural (secrets.md, passwords.md, credentials.md still read as dumps) and +# unlike the singular "token.md", which stays excluded (#3527). Scoped to +# prose extensions only: "tokens.txt" is still a plausible secret store. +_BARE_PLURAL_TOKENS = re.compile(r'(? bool: """A prose/note file (.md/.rst/...) whose stem is a multi-word topic slug is exempt from the generic-keyword drop (#2106). A stem that IS exactly a bare keyword (secrets / token / passwords) is NOT exempt — that still reads as a - credential dump.""" + credential dump. The one exception is bare plural "tokens" (TOKENS.md), + which reads as a design-token reference doc rather than a secret store + (#3527).""" if path.suffix.lower() not in _PROSE_EXTS: return False stem = Path(path.name).stem.lstrip('.') or Path(path.name).stem + if _BARE_PLURAL_TOKENS.fullmatch(stem): + return True return not any(p.fullmatch(stem) for p in _GENERIC_KEYWORD_PATTERNS) diff --git a/tests/test_detect.py b/tests/test_detect.py index 3a2635769..22099029a 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -3412,6 +3412,40 @@ def test_sensitive_bare_keyword_prose_still_dropped(): assert not _is_sensitive(Path("token-lifecycle.md")) # multi-word slug indexed +@pytest.mark.parametrize("path", [ + "TOKENS.md", + "tokens.md", + "tokens.rst", +]) +def test_sensitive_bare_plural_tokens_prose_indexed(path): + """Bare plural "tokens" in a prose file is a design-token reference doc, + not a credential dump — unlike "token.md" (singular) or "secrets.md" + (another keyword's bare plural), which still read as dumps (#3527).""" + from graphify.detect import _is_sensitive + assert not _is_sensitive(Path(path)) + + +def test_sensitive_bare_plural_tokens_still_flagged_outside_prose(): + """The plural exemption is scoped to prose extensions only — "tokens.txt" + is still a plausible secret store and stays excluded (#3527).""" + from graphify.detect import _is_sensitive + assert _is_sensitive(Path("tokens.txt")) + assert _is_sensitive(Path("tokens.json")) + + +@pytest.mark.parametrize("path", [ + "app/lib/theme/shell_tokens.dart", + "src/design/tokens.ts", + "src/hard-tokens.ts", +]) +def test_sensitive_design_token_source_files_indexed(path): + """Genuine design-token source files (.dart/.ts) are graphable source and + exempt from the generic-keyword drop regardless of the bare/plural rules + above — they were the headline repro in #3527.""" + from graphify.detect import _is_sensitive + assert not _is_sensitive(Path(path)) + + # ── #2232 / #2184: committed dotenv templates (.env.example etc.) are graphable ── @pytest.mark.parametrize("path", [