From c46241a751610d82a4c6733e20d472da6888041f Mon Sep 17 00:00:00 2001 From: ayushcodes10 Date: Mon, 14 Sep 2026 22:39:17 +0530 Subject: [PATCH] Use the canonical Unicode aware slugifier for id prefixes Fixes issue 3352. _id_prefixes reimplemented id segment slugification with its own ASCII only regex, so any path segment made of non Latin characters (Korean, CJK, Cyrillic and so on) collapsed to nothing instead of being preserved. _defines_id, the only real caller, uses these prefixes to recognize when a node's own source_file is the file its id encodes, and _collision_rank already implements the definer wins rule this issue asks for. With the wrong prefixes, a node minted from a non ASCII path never matched, so a genuine defining page lost the definer wins tiebreak and the survivor fell through to plain arrival order, exactly the symptom this issue reports. Swapping to normalize_id, the same recipe every real extractor mints an id with, makes the reconstructed prefixes match the ids that are actually minted for such a file. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017qfdzgbA5KedGEjD1AayNh (cherry picked from commit e618f332f6a9ba3b84b5f4674ec16bf009fa2b1d) --- graphify/dedup.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/graphify/dedup.py b/graphify/dedup.py index c1dca1d6c..7c74371ff 100644 --- a/graphify/dedup.py +++ b/graphify/dedup.py @@ -266,7 +266,6 @@ def _is_code(node: dict) -> bool: # ── ID collisions ───────────────────────────────────────────────────────────── -_ID_SEGMENT = re.compile(r"[^a-z0-9]+") _EXTENSION = re.compile(r"\.[^./]+$") @@ -277,10 +276,17 @@ def _id_prefixes(source_file: str) -> set[str]: path, each segment slugified and joined with ``_``. Every trailing slice of the path counts as a prefix: the stored path may be absolute or repo-relative, and graphs built under the pre-#1504 scheme keyed off the bare filename stem. + + Each segment is slugified with ``normalize_id`` (#3352), the same + Unicode-aware casefold-then-NFKC-then-``[^\\w]+`` recipe every real ID is + minted with. An ASCII-only slug here silently dropped every non-Latin + character (Korean, CJK, Cyrillic, ...) instead of preserving it, so a + node's own defining file was never recognized as the file that ID + encodes and the definer-wins collision rule (see ``_defines_id``, + ``_collision_rank``) fell through to arrival-order for any such path. """ stem = _EXTENSION.sub("", source_file.replace("\\", "/")) - segments = [s for s in (_ID_SEGMENT.sub("_", p.casefold()).strip("_") - for p in stem.split("/")) if s] + segments = [s for s in (normalize_id(p) for p in stem.split("/")) if s] return {"_".join(segments[i:]) for i in range(len(segments))}