From c4d6b412cabf422bbac72874933e43e196568690 Mon Sep 17 00:00:00 2001 From: Sir Phillip Tubell Date: Thu, 11 Jun 2026 23:19:57 -0400 Subject: [PATCH] perf: fix O(n^2) -> O(n) LSH neighbor lookup in dedup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaced O(n) linear scan `next((n for n in candidates if n["id"] == neighbor_id))` with O(1) dict lookup via pre-built `candidates_by_id`. Also pre-caches `_norm()` results in `norm_cache` to avoid recomputing per inner iteration. For a 36k-file codebase (~100k high-entropy candidates) this reduces the pass-2 loop from O(n^2*B) (~30–100B iterations) to O(n*B), eliminating the multi-minute CPU hang after AST extraction completes. --- graphify/dedup.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/graphify/dedup.py b/graphify/dedup.py index e37b4c73..fd05797b 100644 --- a/graphify/dedup.py +++ b/graphify/dedup.py @@ -213,19 +213,26 @@ def deduplicate_entities( if len(candidates) >= 2: lsh = MinHashLSH(threshold=_LSH_THRESHOLD, num_perm=_NUM_PERM) minhashes: dict[str, MinHash] = {} + # Pre-build O(1) lookup structures so the query loop below doesn't scan + # the candidates list linearly for every LSH neighbor (was O(n²×B)). + candidates_by_id: dict[str, dict] = {} + norm_cache: dict[str, str] = {} for node in candidates: - norm_label = _norm(node.get("label", node.get("id", ""))) - m = _make_minhash(norm_label) - minhashes[node["id"]] = m + node_id = node["id"] + candidates_by_id[node_id] = node + nl = _norm(node.get("label", node.get("id", ""))) + norm_cache[node_id] = nl + m = _make_minhash(nl) + minhashes[node_id] = m try: - lsh.insert(node["id"], m) + lsh.insert(node_id, m) except ValueError: pass # duplicate key in LSH — already inserted for node in candidates: node_id = node["id"] - norm_label = _norm(node.get("label", node.get("id", ""))) + norm_label = norm_cache[node_id] neighbors = lsh.query(minhashes[node_id]) for neighbor_id in neighbors: @@ -234,11 +241,11 @@ def deduplicate_entities( if uf.find(node_id) == uf.find(neighbor_id): continue - neighbor = next((n for n in candidates if n["id"] == neighbor_id), None) + neighbor = candidates_by_id.get(neighbor_id) if neighbor is None: continue - neighbor_norm = _norm(neighbor.get("label", neighbor.get("id", ""))) + neighbor_norm = norm_cache.get(neighbor_id) or _norm(neighbor.get("label", neighbor.get("id", ""))) score = JaroWinkler.normalized_similarity(norm_label, neighbor_norm) * 100 if _is_variant_pair(norm_label, neighbor_norm):