Wire deduplicate_entities into build() and build_merge()

This commit is contained in:
Safi
2026-05-04 18:02:38 +01:00
parent 34380434c4
commit ec413d5233
2 changed files with 28 additions and 3 deletions
+11 -3
View File
@@ -116,17 +116,19 @@ def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph:
return G
def build(extractions: list[dict], *, directed: bool = False) -> nx.Graph:
def build(extractions: list[dict], *, directed: bool = False, dedup: bool = True) -> nx.Graph:
"""Merge multiple extraction results into one graph.
directed=True produces a DiGraph that preserves edge direction (source→target).
directed=False (default) produces an undirected Graph for backward compatibility.
dedup=True (default) runs entity deduplication before building the graph.
Extractions are merged in order. For nodes with the same ID, the last
extraction's attributes win (NetworkX add_node overwrites). Pass AST
results before semantic results so semantic labels take precedence, or
reverse the order if you prefer AST source_location precision to win.
"""
from graphify.dedup import deduplicate_entities
combined: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0}
for ext in extractions:
combined["nodes"].extend(ext.get("nodes", []))
@@ -134,6 +136,10 @@ def build(extractions: list[dict], *, directed: bool = False) -> nx.Graph:
combined["hyperedges"].extend(ext.get("hyperedges", []))
combined["input_tokens"] += ext.get("input_tokens", 0)
combined["output_tokens"] += ext.get("output_tokens", 0)
if dedup and combined["nodes"]:
combined["nodes"], combined["edges"] = deduplicate_entities(
combined["nodes"], combined["edges"], communities={}
)
return build_from_json(combined, directed=directed)
@@ -194,6 +200,7 @@ def build_merge(
prune_sources: list[str] | None = None,
*,
directed: bool = False,
dedup: bool = True,
) -> nx.Graph:
"""Load existing graph.json, merge new chunks into it, and save back.
@@ -219,7 +226,7 @@ def build_merge(
base = []
all_chunks = base + list(new_chunks)
G = build(all_chunks, directed=directed)
G = build(all_chunks, directed=directed, dedup=dedup)
# Prune nodes from deleted source files
if prune_sources:
@@ -232,7 +239,8 @@ def build_merge(
print(f"[graphify] Pruned {len(to_remove)} node(s) from deleted sources.", file=sys.stderr)
# Safety check: refuse to shrink the graph silently (#479)
if graph_path.exists():
# Skip when dedup or prune_sources is active — shrinkage is intentional there.
if graph_path.exists() and not dedup and not prune_sources:
existing_n = len(existing_nodes)
new_n = G.number_of_nodes()
if new_n < existing_n:
+17
View File
@@ -118,3 +118,20 @@ def test_dedup_llm_flag_accepted():
edges = []
result_nodes, _ = deduplicate_entities(nodes, edges, communities={}, dedup_llm_backend=None)
assert len(result_nodes) == 2
# ── build integration ─────────────────────────────────────────────────────────
def test_build_calls_dedup():
"""build() should deduplicate near-identical nodes across extractions."""
from graphify.build import build
chunk1 = {
"nodes": [{"id": "graphextractor", "label": "GraphExtractor", "source_file": "a.py"}],
"edges": [],
}
chunk2 = {
"nodes": [{"id": "graph_extractor", "label": "Graph Extractor", "source_file": "b.py"}],
"edges": [],
}
G = build([chunk1, chunk2])
assert G.number_of_nodes() == 1