From 0c01da19323913024ae0e45cae54319459410fad Mon Sep 17 00:00:00 2001 From: stupidprogrammer4 Date: Thu, 30 Jul 2026 21:58:07 +0330 Subject: [PATCH] perf(dedup): remove O(nodes x components) scan from remap construction deduplicate_entities() rebuilt its remap table by re-scanning the whole unique_nodes list once per merged union-find component, costing O(nodes x components) -- roughly 137M membership checks on a 50k-node corpus with 2.5k merged components, about a third of dedup wall-clock. Build an id -> (position, node) index once and slice each component out of it instead: O(members log members) per component. The index carries the enumeration position, not just the node, because _pick_winner() selects via min(), which returns the first minimum -- so ties (equal chunk-suffix status and equal id length) are resolved by list order. Sorting the group by id instead changed 179 of 2358 survivors on a tie-heavy corpus. Sorting by position reproduces unique_nodes order exactly, leaving survivors and edges byte-identical. Benchmark, median of 5 runs: nodes before after speedup 5000 2.37s 2.29s 1.03x 20000 10.58s 9.70s 1.09x 50000 31.72s 21.92s 1.45x Isolated remap loop at 50k nodes / 2.5k components: 6.838s -> 0.019s (370x). Merge counts and output node counts are identical at every scale. This does not address the pre-existing arrival-order dependence of the component structure itself, which originates in the fuzzy-merge loop's uf.find() short-circuit and is out of scope here. Co-Authored-By: Claude Opus 5 (1M context) --- graphify/dedup.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/graphify/dedup.py b/graphify/dedup.py index bc2f1c418..e9ce0398f 100644 --- a/graphify/dedup.py +++ b/graphify/dedup.py @@ -574,10 +574,26 @@ def deduplicate_entities( components = uf.components() remap: dict[str, str] = {} + # id -> (position, node), built once. Previously each component re-scanned + # the whole unique_nodes list, making remap construction O(nodes x + # components) — 31% of dedup wall-clock on a 50k-node corpus. + # The position is carried so group_nodes keeps unique_nodes order: _pick_winner + # resolves ties (equal chunk-suffix status and equal id length) via min(), + # which returns the first minimum, so reordering here would silently change + # which node survives. + nodes_by_id: dict[str, tuple[int, dict]] = { + n["id"]: (i, n) for i, n in enumerate(unique_nodes) + } + for root, members in components.items(): if len(members) == 1: continue - group_nodes = [n for n in unique_nodes if n["id"] in members] + group_nodes = [ + n for _, n in sorted( + (nodes_by_id[m] for m in members if m in nodes_by_id), + key=lambda pair: pair[0], + ) + ] winner = _pick_winner(group_nodes) if group_nodes else {"id": root} winner_id = winner["id"] for member in members: