From 86109e9f272606abff3c4c08beaef01b5f6138a9 Mon Sep 17 00:00:00 2001 From: szgnewGh Date: Fri, 22 May 2026 21:26:27 +0800 Subject: [PATCH] fix: CJK/Unicode labels silently skipped in _norm/_norm_label dedup (follow-up to #811) (#937) --- graphify/build.py | 5 +++-- graphify/dedup.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/graphify/build.py b/graphify/build.py index 68f50bc9f..582f3f2ac 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -228,8 +228,9 @@ def build( def _norm_label(label: str) -> str: - """Canonical dedup key — lowercase, alphanumeric only.""" - return re.sub(r"[^a-z0-9 ]", "", label.lower()).strip() + """Canonical dedup key — Unicode-aware, preserves CJK/word characters.""" + label = unicodedata.normalize("NFKC", label) + return re.sub(r"[\W_ ]+", " ", label.casefold(), flags=re.UNICODE).strip() def deduplicate_by_label(nodes: list[dict], edges: list[dict]) -> tuple[list[dict], list[dict]]: diff --git a/graphify/dedup.py b/graphify/dedup.py index 5c15f33f5..dc177fb8f 100644 --- a/graphify/dedup.py +++ b/graphify/dedup.py @@ -6,6 +6,7 @@ Jaro-Winkler verification → same-community boost → union-find merge. from __future__ import annotations import math import re +import unicodedata from collections import defaultdict from datasketch import MinHash, MinHashLSH @@ -15,8 +16,9 @@ from rapidfuzz.distance import JaroWinkler # ── helpers ─────────────────────────────────────────────────────────────────── def _norm(label: str) -> str: - """Lowercase + collapse non-alphanumeric runs to space.""" - return re.sub(r"[^a-z0-9]+", " ", label.lower()).strip() + """Lowercase + collapse non-alphanumeric runs to space (Unicode-aware).""" + label = unicodedata.normalize("NFKC", label) + return re.sub(r"[\W_]+", " ", label.casefold(), flags=re.UNICODE).strip() def _entropy(label: str) -> float: