diff --git a/graphify/build.py b/graphify/build.py index 68f50bc9f..582f3f2ac 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -228,8 +228,9 @@ def build( def _norm_label(label: str) -> str: - """Canonical dedup key — lowercase, alphanumeric only.""" - return re.sub(r"[^a-z0-9 ]", "", label.lower()).strip() + """Canonical dedup key — Unicode-aware, preserves CJK/word characters.""" + label = unicodedata.normalize("NFKC", label) + return re.sub(r"[\W_ ]+", " ", label.casefold(), flags=re.UNICODE).strip() def deduplicate_by_label(nodes: list[dict], edges: list[dict]) -> tuple[list[dict], list[dict]]: diff --git a/graphify/dedup.py b/graphify/dedup.py index 5c15f33f5..dc177fb8f 100644 --- a/graphify/dedup.py +++ b/graphify/dedup.py @@ -6,6 +6,7 @@ Jaro-Winkler verification → same-community boost → union-find merge. from __future__ import annotations import math import re +import unicodedata from collections import defaultdict from datasketch import MinHash, MinHashLSH @@ -15,8 +16,9 @@ from rapidfuzz.distance import JaroWinkler # ── helpers ─────────────────────────────────────────────────────────────────── def _norm(label: str) -> str: - """Lowercase + collapse non-alphanumeric runs to space.""" - return re.sub(r"[^a-z0-9]+", " ", label.lower()).strip() + """Lowercase + collapse non-alphanumeric runs to space (Unicode-aware).""" + label = unicodedata.normalize("NFKC", label) + return re.sub(r"[\W_]+", " ", label.casefold(), flags=re.UNICODE).strip() def _entropy(label: str) -> float: