diff --git a/graphify/dedup.py b/graphify/dedup.py index e37b4c73..fd05797b 100644 --- a/graphify/dedup.py +++ b/graphify/dedup.py @@ -213,19 +213,26 @@ def deduplicate_entities( if len(candidates) >= 2: lsh = MinHashLSH(threshold=_LSH_THRESHOLD, num_perm=_NUM_PERM) minhashes: dict[str, MinHash] = {} + # Pre-build O(1) lookup structures so the query loop below doesn't scan + # the candidates list linearly for every LSH neighbor (was O(n²×B)). + candidates_by_id: dict[str, dict] = {} + norm_cache: dict[str, str] = {} for node in candidates: - norm_label = _norm(node.get("label", node.get("id", ""))) - m = _make_minhash(norm_label) - minhashes[node["id"]] = m + node_id = node["id"] + candidates_by_id[node_id] = node + nl = _norm(node.get("label", node.get("id", ""))) + norm_cache[node_id] = nl + m = _make_minhash(nl) + minhashes[node_id] = m try: - lsh.insert(node["id"], m) + lsh.insert(node_id, m) except ValueError: pass # duplicate key in LSH — already inserted for node in candidates: node_id = node["id"] - norm_label = _norm(node.get("label", node.get("id", ""))) + norm_label = norm_cache[node_id] neighbors = lsh.query(minhashes[node_id]) for neighbor_id in neighbors: @@ -234,11 +241,11 @@ def deduplicate_entities( if uf.find(node_id) == uf.find(neighbor_id): continue - neighbor = next((n for n in candidates if n["id"] == neighbor_id), None) + neighbor = candidates_by_id.get(neighbor_id) if neighbor is None: continue - neighbor_norm = _norm(neighbor.get("label", neighbor.get("id", ""))) + neighbor_norm = norm_cache.get(neighbor_id) or _norm(neighbor.get("label", neighbor.get("id", ""))) score = JaroWinkler.normalized_similarity(norm_label, neighbor_norm) * 100 if _is_variant_pair(norm_label, neighbor_norm):