lee/graphify
1
0
Fork 0
mirror of https://github.com/safishamsi/graphify.git synced 2026-09-25 15:05:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: CJK/Unicode labels silently skipped in _norm/_norm_label dedup (follow-up to #811) (#937)

Browse Source
This commit is contained in:
szgnewGh
2026-05-22 14:26:27 +01:00
committed by GitHub
parent 6efd06c54d
commit 86109e9f27
2 changed files with 7 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
graphify/build.py
+3 -2
View File
@@ -228,8 +228,9 @@ def build(
def _norm_label(label: str) -> str:
"""Canonical dedup key — lowercase, alphanumeric only."""
return re.sub(r"[^a-z0-9 ]", "", label.lower()).strip()
"""Canonical dedup key — Unicode-aware, preserves CJK/word characters."""
label = unicodedata.normalize("NFKC", label)
return re.sub(r"[\W_ ]+", " ", label.casefold(), flags=re.UNICODE).strip()
def deduplicate_by_label(nodes: list[dict], edges: list[dict]) -> tuple[list[dict], list[dict]]:
graphify/dedup.py
+4 -2
View File
@@ -6,6 +6,7 @@ Jaro-Winkler verification → same-community boost → union-find merge.
from __future__ import annotations
import math
import re
import unicodedata
from collections import defaultdict
from datasketch import MinHash, MinHashLSH
@@ -15,8 +16,9 @@ from rapidfuzz.distance import JaroWinkler
# ── helpers ───────────────────────────────────────────────────────────────────
def _norm(label: str) -> str:
"""Lowercase + collapse non-alphanumeric runs to space."""
return re.sub(r"[^a-z0-9]+", " ", label.lower()).strip()
"""Lowercase + collapse non-alphanumeric runs to space (Unicode-aware)."""
label = unicodedata.normalize("NFKC", label)
return re.sub(r"[\W_]+", " ", label.casefold(), flags=re.UNICODE).strip()
def _entropy(label: str) -> float: