diff --git a/graphify/build.py b/graphify/build.py index 92c05d2ff..d247b2962 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -1559,11 +1559,79 @@ def _load_existing_graph(graph_path: Path) -> "tuple[list, list, list, bool] | N ) +def _tier_replacement_sources( + chunks: "Iterable[dict]", + root: "str | Path | None" = None, + ast_sources: "Iterable[str | Path] | None" = None, +) -> tuple[set[str], set[str]]: + """Compute (new_ast_sources, new_sem_sources) for tier-scoped replacement. + + #3411: AST replacement ownership is derived from explicit extraction + provenance (ast_sources or chunk-level "extracted_sources"), so cross-file + stub nodes emitted by extractors (e.g. .sln project stubs or ProjectReference + stubs) do not pollute the replacement set and wipe the target project's + nodes/edges. Falls back to AST node source_file only when no provenance is + provided. + """ + explicit_ast_sources: set[str] = set() + if ast_sources is not None: + for s in ast_sources: + if s: + explicit_ast_sources.add(str(s)) + for ch in chunks: + if isinstance(ch, dict): + for s in (ch.get("extracted_sources") or []): + if s: + explicit_ast_sources.add(str(s)) + + new_ast_sources: set[str] = set() + new_sem_sources: set[str] = set() + + if explicit_ast_sources: + for sf in explicit_ast_sources: + new_ast_sources.add(sf) + norm = _norm_source_file(sf, root) + if norm: + new_ast_sources.add(norm) + else: + for ch in chunks: + if not isinstance(ch, dict): + continue + for n in ch.get("nodes", []): + if not isinstance(n, dict): + continue + sf = n.get("source_file") + if not sf or not _is_ast_tier(n): + continue + new_ast_sources.add(sf) + norm = _norm_source_file(sf, root) + if norm: + new_ast_sources.add(norm) + + for ch in chunks: + if not isinstance(ch, dict): + continue + for n in ch.get("nodes", []): + if not isinstance(n, dict): + continue + sf = n.get("source_file") + if not sf or _is_ast_tier(n): + continue + new_sem_sources.add(sf) + norm = _norm_source_file(sf, root) + if norm: + new_sem_sources.add(norm) + + return new_ast_sources, new_sem_sources + + def merge_raw_extraction( new: dict, graph_path: str | Path, prune_sources: "list[str] | None" = None, root: "str | Path | None" = None, + *, + ast_sources: "Iterable[str | Path] | None" = None, ) -> dict: """Merge the existing raw graph.json forward into a fresh raw extraction (the ``extract --no-cluster`` incremental path, #2169). @@ -1572,10 +1640,13 @@ def merge_raw_extraction( clustered incremental paths can't drift: - sources re-extracted this run REPLACE their prior contribution PER TIER - (#2333/#2336): existing nodes/edges/hyperedges owned by them are dropped + (#2333/#2336, #3411): existing nodes/edges/hyperedges owned by them are dropped only when the new extraction contains the same tier (AST vs semantic, per :func:`_is_ast_tier`) for that source, matched in both raw and - :func:`_norm_source_file` form (#1007); + :func:`_norm_source_file` form (#1007). AST replacement ownership is derived + from explicit extraction provenance (``ast_sources`` or chunk-level + ``extracted_sources``), falling back to AST node source_file only when no + provenance is provided (#3411); - ``prune_sources`` (deleted / excluded / graph-stale files) are dropped, with the ``_abs_identity`` third-form fallback (#2012), and "replace" wins over a contradictory "delete" of a re-extracted source (#1796); @@ -1602,23 +1673,15 @@ def merge_raw_extraction( else _infer_merge_root(graph_path) ) - # Tier-scoped replace, mirroring build_merge (#2333/#2336, COEXIST): a + # Tier-scoped replace, mirroring build_merge (#2333/#2336, COEXIST, #3411): a # source re-extracted this run replaces only the tier(s) actually present # in the new extraction, so an AST-only re-extract keeps the file's - # semantic layer and vice versa. - new_ast_sources: set[str] = set() - new_sem_sources: set[str] = set() - for n in new.get("nodes", []): - if not isinstance(n, dict): - continue - sf = n.get("source_file") - if not sf: - continue - tier_sources = new_ast_sources if _is_ast_tier(n) else new_sem_sources - tier_sources.add(sf) - norm = _norm_source_file(sf, _eff_root) - if norm: - tier_sources.add(norm) + # semantic layer and vice versa. AST replacement ownership is derived from + # explicit extraction provenance (ast_sources or "extracted_sources"), + # falling back to AST node source_files only when no provenance is provided. + new_ast_sources, new_sem_sources = _tier_replacement_sources( + [new], root=_eff_root, ast_sources=ast_sources + ) new_sources: set[str] = new_ast_sources | new_sem_sources # "Replace" wins over a contradictory "delete" of the same source (#1796), @@ -1715,6 +1778,7 @@ def build_merge( dedup: bool = True, dedup_llm_backend: str | None = None, root: str | Path | None = None, + ast_sources: "Iterable[str | Path] | None" = None, ) -> nx.Graph: """Load existing graph.json and return it merged with ``new_chunks``. @@ -1722,13 +1786,16 @@ def build_merge( ``export.to_json(G, communities, graph_path, force=True)`` after clustering. ``graph_path`` is read-only here. - Re-extracted files REPLACE their prior contribution per tier (#2333/#2336): + Re-extracted files REPLACE their prior contribution per tier (#2333/#2336, #3411): a source_file present in new_chunks has its existing nodes/edges dropped for each tier (AST vs semantic, per :func:`_is_ast_tier`) the new chunks actually contain, so a changed file's stale nodes/edges don't accumulate - while a one-tier re-extract keeps the other tier's layer intact. Files - absent from new_chunks are preserved unchanged; deleted files are removed - via prune_sources (tier-blind). Safe to call repeatedly. + while a one-tier re-extract keeps the other tier's layer intact. AST replacement + ownership is derived from explicit extraction provenance (``ast_sources`` or + chunk-level ``extracted_sources``), falling back to AST node source_file only + when no provenance is provided (#3411). Files absent from new_chunks are + preserved unchanged; deleted files are removed via prune_sources (tier-blind). + Safe to call repeatedly. root: if given, absolute source_file paths in new_chunks are made relative (#932). directed: if None (default), honor the on-disk graph's own ``directed`` flag when one exists, so an incremental merge can't silently flip a directed @@ -1781,19 +1848,17 @@ def build_merge( # chunk used to delete the file's AST headings). Which tier a NEW chunk # item belongs to is read via _is_ast_tier (existing items were stamped by # _load_existing_graph above). + # + # #3411: AST replacement ownership is derived from explicit extraction + # provenance (ast_sources or chunk-level "extracted_sources"), so cross-file + # stub nodes emitted by extractors (e.g. .sln project stubs or ProjectReference + # stubs) do not pollute the replacement set and wipe the target project's + # nodes/edges. Falls back to AST node source_file only when no provenance is + # provided. _replace_root = _eff_root - new_ast_sources: set[str] = set() - new_sem_sources: set[str] = set() - for ch in new_chunks: - for n in ch.get("nodes", []): - sf = n.get("source_file") - if not sf: - continue - tier_sources = new_ast_sources if _is_ast_tier(n) else new_sem_sources - tier_sources.add(sf) - norm = _norm_source_file(sf, _replace_root) - if norm: - tier_sources.add(norm) + new_ast_sources, new_sem_sources = _tier_replacement_sources( + new_chunks, root=_replace_root, ast_sources=ast_sources + ) new_sources: set[str] = new_ast_sources | new_sem_sources # True on-disk baseline for the #479 shrink accounting at the end (#2497): # the rebind below removes the re-extracted sources' old nodes from diff --git a/graphify/cli.py b/graphify/cli.py index efede012f..400cf463d 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -4091,6 +4091,7 @@ def dispatch_command(cmd: str) -> None: "hyperedges": list(sem_result.get("hyperedges", [])), "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0), "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0), + "extracted_sources": list(ast_result.get("extracted_sources", [])), } graph_json_path = graphify_out / "graph.json" diff --git a/graphify/extract.py b/graphify/extract.py index 8c358761b..32c59484d 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -7627,6 +7627,10 @@ def extract( # manifest does not freeze them as processed (#2543). Callers that # only read nodes/edges ignore this key. "failed_sources": _failed_sources, + # Surfaces the actual dispatched source paths so build_merge / + # merge_raw_extraction know which files were genuinely re-extracted + # rather than guessing ownership from node["source_file"] (#3411). + "extracted_sources": [str(p) for p in paths], } diff --git a/tests/test_build.py b/tests/test_build.py index b01f5439a..a19ca48e9 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -1713,3 +1713,273 @@ def test_derive_prune_root_recovers_root_from_posix_absolute_prune_sources(): assert _derive_prune_root( ["/home/ci/build/repo/docs/a.md"], stored ) == "/home/ci/build/repo" + + +# ── #3411: Cross-file stub nodes must not pollute AST replacement set ──────── + + +def test_build_merge_sln_stub_does_not_replace_referenced_csproj(tmp_path): + """#3411: Re-extracting a .sln must replace only the .sln's AST contribution. + Cross-file project stubs (source_file='A/A.csproj') in the .sln chunk must not + cause A/A.csproj's PackageReference/TargetFramework nodes and edges to be wiped.""" + import networkx as nx + + root = tmp_path / "corpus" + root.mkdir() + graph_path = tmp_path / "graph.json" + + # Initial full extraction: + # 1) A.csproj with framework, package reference, and internal edges + # 2) A.sln containing A.csproj + c_proj = { + "nodes": [ + {"id": "a_a_csproj", "label": "A.csproj", "file_type": "code", + "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"}, + {"id": "framework_net8_0", "label": "net8.0", "file_type": "concept", + "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"}, + {"id": "nuget_newtonsoft_json", "label": "Newtonsoft.Json (13.0.3)", + "file_type": "code", "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"}, + ], + "edges": [ + {"source": "a_a_csproj", "target": "framework_net8_0", "relation": "references", + "confidence": "EXTRACTED", "source_file": "A/A.csproj", "weight": 1.0, "_origin": "ast"}, + {"source": "a_a_csproj", "target": "nuget_newtonsoft_json", "relation": "imports", + "confidence": "EXTRACTED", "source_file": "A/A.csproj", "weight": 1.0, "_origin": "ast"}, + ], + "extracted_sources": [str(root / "A" / "A.csproj")], + } + sln_initial = { + "nodes": [ + {"id": "a_sln", "label": "A.sln", "file_type": "code", + "source_file": "A.sln", "source_location": None, "_origin": "ast"}, + {"id": "a_a_csproj", "label": "A", "file_type": "code", + "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"}, + ], + "edges": [ + {"source": "a_sln", "target": "a_a_csproj", "relation": "contains", + "confidence": "EXTRACTED", "source_file": "A.sln", "weight": 1.0, "_origin": "ast"}, + ], + "extracted_sources": [str(root / "A.sln")], + } + G0 = build([c_proj, sln_initial], dedup=False, root=root) + graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8") + + # Incremental update: ONLY A.sln is modified and re-extracted. + # extract_sln produces the solution node AND a cross-file stub for A/A.csproj. + sln_reextracted = { + "nodes": [ + {"id": "a_sln", "label": "A.sln", "file_type": "code", + "source_file": "A.sln", "source_location": None, "_origin": "ast"}, + {"id": "a_a_csproj", "label": "A", "file_type": "code", + "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"}, + ], + "edges": [ + {"source": "a_sln", "target": "a_a_csproj", "relation": "contains", + "confidence": "EXTRACTED", "source_file": "A.sln", "weight": 1.0, "_origin": "ast"}, + ], + # Explicit provenance: only A.sln was actually extracted + "extracted_sources": [str(root / "A.sln")], + } + + G1 = build_merge([sln_reextracted], graph_path, dedup=False, root=root) + + node_ids = set(G1.nodes()) + # 1. Solution node and project stub node are present + assert "a_sln" in node_ids, "Solution node must be present" + assert "a_a_csproj" in node_ids, "Project node must be present" + # 2. PackageReference and TargetFramework nodes of A.csproj must SURVIVE + assert "framework_net8_0" in node_ids, ( + "#3411 regression: TargetFramework node of referenced .csproj was wiped by .sln re-extraction" + ) + assert "nuget_newtonsoft_json" in node_ids, ( + "#3411 regression: PackageReference node of referenced .csproj was wiped by .sln re-extraction" + ) + # 3. Edges must SURVIVE + assert G1.has_edge("a_sln", "a_a_csproj"), "sln contains edge must be present" + assert G1.has_edge("a_a_csproj", "framework_net8_0"), ( + "#3411 regression: references edge to TargetFramework was wiped" + ) + assert G1.has_edge("a_a_csproj", "nuget_newtonsoft_json"), ( + "#3411 regression: imports edge to PackageReference was wiped" + ) + + +def test_build_merge_project_reference_stub_does_not_replace_referenced_project(tmp_path): + """#3411: Re-extracting B.csproj (which has a ProjectReference to A.csproj) must + replace only B.csproj's AST contribution. A.csproj's PackageReference and + TargetFramework nodes must remain intact.""" + import networkx as nx + + root = tmp_path / "corpus" + root.mkdir() + graph_path = tmp_path / "graph.json" + + # Initial graph: A.csproj with package & framework; B.csproj referencing A.csproj + c_proj_a = { + "nodes": [ + {"id": "a_csproj", "label": "A.csproj", "file_type": "code", + "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"}, + {"id": "framework_net8_0", "label": "net8.0", "file_type": "concept", + "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"}, + {"id": "nuget_pkg_a", "label": "PackageA (1.0.0)", "file_type": "code", + "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"}, + ], + "edges": [ + {"source": "a_csproj", "target": "framework_net8_0", "relation": "references", + "confidence": "EXTRACTED", "source_file": "A/A.csproj", "weight": 1.0, "_origin": "ast"}, + {"source": "a_csproj", "target": "nuget_pkg_a", "relation": "imports", + "confidence": "EXTRACTED", "source_file": "A/A.csproj", "weight": 1.0, "_origin": "ast"}, + ], + "extracted_sources": [str(root / "A" / "A.csproj")], + } + c_proj_b = { + "nodes": [ + {"id": "b_csproj", "label": "B.csproj", "file_type": "code", + "source_file": "B/B.csproj", "source_location": None, "_origin": "ast"}, + {"id": "b_old_symbol", "label": "old_fn", "file_type": "code", + "source_file": "B/B.csproj", "source_location": "L10", "_origin": "ast"}, + {"id": "a_csproj", "label": "A.csproj", "file_type": "code", + "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"}, + ], + "edges": [ + {"source": "b_csproj", "target": "b_old_symbol", "relation": "contains", + "confidence": "EXTRACTED", "source_file": "B/B.csproj", "weight": 1.0, "_origin": "ast"}, + {"source": "b_csproj", "target": "a_csproj", "relation": "imports", + "confidence": "EXTRACTED", "source_file": "B/B.csproj", "weight": 1.0, "_origin": "ast"}, + ], + "extracted_sources": [str(root / "B" / "B.csproj")], + } + G0 = build([c_proj_a, c_proj_b], dedup=False, root=root) + graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8") + + # Re-extract ONLY B.csproj (b_old_symbol replaced by b_new_symbol; ProjectReference stub to A.csproj emitted) + c_proj_b_reextracted = { + "nodes": [ + {"id": "b_csproj", "label": "B.csproj", "file_type": "code", + "source_file": "B/B.csproj", "source_location": None, "_origin": "ast"}, + {"id": "b_new_symbol", "label": "new_fn", "file_type": "code", + "source_file": "B/B.csproj", "source_location": "L12", "_origin": "ast"}, + {"id": "a_csproj", "label": "A.csproj", "file_type": "code", + "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"}, + ], + "edges": [ + {"source": "b_csproj", "target": "b_new_symbol", "relation": "contains", + "confidence": "EXTRACTED", "source_file": "B/B.csproj", "weight": 1.0, "_origin": "ast"}, + {"source": "b_csproj", "target": "a_csproj", "relation": "imports", + "confidence": "EXTRACTED", "source_file": "B/B.csproj", "weight": 1.0, "_origin": "ast"}, + ], + "extracted_sources": [str(root / "B" / "B.csproj")], + } + + G1 = build_merge([c_proj_b_reextracted], graph_path, dedup=False, root=root) + + node_ids = set(G1.nodes()) + # 1. B.csproj was replaced properly + assert "b_new_symbol" in node_ids, "New symbol in B.csproj must be present" + assert "b_old_symbol" not in node_ids, "Stale symbol in B.csproj must be dropped" + # 2. A.csproj package & framework nodes must NOT be dropped + assert "nuget_pkg_a" in node_ids, ( + "#3411 regression: PackageReference in referenced A.csproj was wiped when B.csproj was re-extracted" + ) + assert "framework_net8_0" in node_ids, ( + "#3411 regression: TargetFramework in referenced A.csproj was wiped when B.csproj was re-extracted" + ) + # 3. All edges survive + assert G1.has_edge("a_csproj", "nuget_pkg_a") + assert G1.has_edge("a_csproj", "framework_net8_0") + assert G1.has_edge("b_csproj", "a_csproj") + + +def test_merge_raw_extraction_cross_file_stub_parity(tmp_path): + """#3411: merge_raw_extraction shares the same explicit extraction provenance + scoping as build_merge.""" + import networkx as nx + from graphify.build import merge_raw_extraction + + root = tmp_path / "corpus" + root.mkdir() + graph_path = tmp_path / "graph.json" + + # Initial graph on disk + c_proj = { + "nodes": [ + {"id": "a_csproj", "label": "A.csproj", "file_type": "code", + "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"}, + {"id": "nuget_pkg", "label": "PackageA", "file_type": "code", + "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"}, + ], + "edges": [ + {"source": "a_csproj", "target": "nuget_pkg", "relation": "imports", + "confidence": "EXTRACTED", "source_file": "A/A.csproj", "weight": 1.0, "_origin": "ast"}, + ], + } + G0 = build([c_proj], dedup=False, root=root) + graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8") + + # Re-extract A.sln with cross-file stub for A.csproj + sln_reextracted = { + "nodes": [ + {"id": "a_sln", "label": "A.sln", "file_type": "code", + "source_file": "A.sln", "source_location": None, "_origin": "ast"}, + {"id": "a_csproj", "label": "A", "file_type": "code", + "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"}, + ], + "edges": [ + {"source": "a_sln", "target": "a_csproj", "relation": "contains", + "confidence": "EXTRACTED", "source_file": "A.sln", "weight": 1.0, "_origin": "ast"}, + ], + "extracted_sources": ["A.sln"], + } + + merged = merge_raw_extraction(sln_reextracted, graph_path=graph_path, root=root) + merged_node_ids = {n["id"] for n in merged["nodes"]} + + assert "nuget_pkg" in merged_node_ids, ( + "#3411: raw incremental path wiped PackageReference from referenced .csproj" + ) + assert any( + e.get("source") == "a_csproj" and e.get("target") == "nuget_pkg" + for e in merged["edges"] + ), "#3411: raw incremental path wiped imports edge to PackageReference" + + +def test_build_merge_explicit_ast_sources_argument(tmp_path): + """#3411: build_merge honors ast_sources argument when passed explicitly.""" + import networkx as nx + + root = tmp_path / "corpus" + root.mkdir() + graph_path = tmp_path / "graph.json" + + c_proj = { + "nodes": [ + {"id": "a_csproj", "label": "A.csproj", "file_type": "code", + "source_file": "A/A.csproj", "_origin": "ast"}, + {"id": "nuget_pkg", "label": "PackageA", "file_type": "code", + "source_file": "A/A.csproj", "_origin": "ast"}, + ], + "edges": [ + {"source": "a_csproj", "target": "nuget_pkg", "relation": "imports", + "source_file": "A/A.csproj", "_origin": "ast"}, + ], + } + G0 = build([c_proj], dedup=False, root=root) + graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8") + + # Chunk has nodes without 'extracted_sources' attached to chunk, + # but ast_sources is provided explicitly to build_merge + sln_chunk = { + "nodes": [ + {"id": "a_sln", "label": "A.sln", "file_type": "code", + "source_file": "A.sln", "_origin": "ast"}, + {"id": "a_csproj", "label": "A", "file_type": "code", + "source_file": "A/A.csproj", "_origin": "ast"}, + ], + "edges": [ + {"source": "a_sln", "target": "a_csproj", "relation": "contains", + "source_file": "A.sln", "_origin": "ast"}, + ], + } + + G1 = build_merge([sln_chunk], graph_path, dedup=False, root=root, ast_sources=["A.sln"]) + assert "nuget_pkg" in G1, "ast_sources explicit arg must protect undispatched A.csproj"