diff --git a/graphify/dedup.py b/graphify/dedup.py index bc2f1c418..e9ce0398f 100644 --- a/graphify/dedup.py +++ b/graphify/dedup.py @@ -574,10 +574,26 @@ def deduplicate_entities( components = uf.components() remap: dict[str, str] = {} + # id -> (position, node), built once. Previously each component re-scanned + # the whole unique_nodes list, making remap construction O(nodes x + # components) — 31% of dedup wall-clock on a 50k-node corpus. + # The position is carried so group_nodes keeps unique_nodes order: _pick_winner + # resolves ties (equal chunk-suffix status and equal id length) via min(), + # which returns the first minimum, so reordering here would silently change + # which node survives. + nodes_by_id: dict[str, tuple[int, dict]] = { + n["id"]: (i, n) for i, n in enumerate(unique_nodes) + } + for root, members in components.items(): if len(members) == 1: continue - group_nodes = [n for n in unique_nodes if n["id"] in members] + group_nodes = [ + n for _, n in sorted( + (nodes_by_id[m] for m in members if m in nodes_by_id), + key=lambda pair: pair[0], + ) + ] winner = _pick_winner(group_nodes) if group_nodes else {"id": root} winner_id = winner["id"] for member in members: