Fix incremental merge of cross-file project stubs

(cherry picked from commit 8286a51511)

graphify/build.py

@@ -1559,11 +1559,79 @@ def _load_existing_graph(graph_path: Path) -> "tuple[list, list, list, bool] | N
     )
 
 
+def _tier_replacement_sources(
+    chunks: "Iterable[dict]",
+    root: "str | Path | None" = None,
+    ast_sources: "Iterable[str | Path] | None" = None,
+) -> tuple[set[str], set[str]]:
+    """Compute (new_ast_sources, new_sem_sources) for tier-scoped replacement.
+
+    #3411: AST replacement ownership is derived from explicit extraction
+    provenance (ast_sources or chunk-level "extracted_sources"), so cross-file
+    stub nodes emitted by extractors (e.g. .sln project stubs or ProjectReference
+    stubs) do not pollute the replacement set and wipe the target project's
+    nodes/edges. Falls back to AST node source_file only when no provenance is
+    provided.
+    """
+    explicit_ast_sources: set[str] = set()
+    if ast_sources is not None:
+        for s in ast_sources:
+            if s:
+                explicit_ast_sources.add(str(s))
+    for ch in chunks:
+        if isinstance(ch, dict):
+            for s in (ch.get("extracted_sources") or []):
+                if s:
+                    explicit_ast_sources.add(str(s))
+
+    new_ast_sources: set[str] = set()
+    new_sem_sources: set[str] = set()
+
+    if explicit_ast_sources:
+        for sf in explicit_ast_sources:
+            new_ast_sources.add(sf)
+            norm = _norm_source_file(sf, root)
+            if norm:
+                new_ast_sources.add(norm)
+    else:
+        for ch in chunks:
+            if not isinstance(ch, dict):
+                continue
+            for n in ch.get("nodes", []):
+                if not isinstance(n, dict):
+                    continue
+                sf = n.get("source_file")
+                if not sf or not _is_ast_tier(n):
+                    continue
+                new_ast_sources.add(sf)
+                norm = _norm_source_file(sf, root)
+                if norm:
+                    new_ast_sources.add(norm)
+
+    for ch in chunks:
+        if not isinstance(ch, dict):
+            continue
+        for n in ch.get("nodes", []):
+            if not isinstance(n, dict):
+                continue
+            sf = n.get("source_file")
+            if not sf or _is_ast_tier(n):
+                continue
+            new_sem_sources.add(sf)
+            norm = _norm_source_file(sf, root)
+            if norm:
+                new_sem_sources.add(norm)
+
+    return new_ast_sources, new_sem_sources
+
+
 def merge_raw_extraction(
     new: dict,
     graph_path: str | Path,
     prune_sources: "list[str] | None" = None,
     root: "str | Path | None" = None,
+    *,
+    ast_sources: "Iterable[str | Path] | None" = None,
 ) -> dict:
     """Merge the existing raw graph.json forward into a fresh raw extraction
     (the ``extract --no-cluster`` incremental path, #2169).
@@ -1572,10 +1640,13 @@ def merge_raw_extraction(
     clustered incremental paths can't drift:
 
     - sources re-extracted this run REPLACE their prior contribution PER TIER
-      (#2333/#2336): existing nodes/edges/hyperedges owned by them are dropped
+      (#2333/#2336, #3411): existing nodes/edges/hyperedges owned by them are dropped
       only when the new extraction contains the same tier (AST vs semantic,
       per :func:`_is_ast_tier`) for that source, matched in both raw and
-      :func:`_norm_source_file` form (#1007);
+      :func:`_norm_source_file` form (#1007). AST replacement ownership is derived
+      from explicit extraction provenance (``ast_sources`` or chunk-level
+      ``extracted_sources``), falling back to AST node source_file only when no
+      provenance is provided (#3411);
     - ``prune_sources`` (deleted / excluded / graph-stale files) are dropped,
       with the ``_abs_identity`` third-form fallback (#2012), and "replace" wins
       over a contradictory "delete" of a re-extracted source (#1796);
@@ -1602,23 +1673,15 @@ def merge_raw_extraction(
         else _infer_merge_root(graph_path)
     )
 
-    # Tier-scoped replace, mirroring build_merge (#2333/#2336, COEXIST): a
+    # Tier-scoped replace, mirroring build_merge (#2333/#2336, COEXIST, #3411): a
     # source re-extracted this run replaces only the tier(s) actually present
     # in the new extraction, so an AST-only re-extract keeps the file's
-    # semantic layer and vice versa.
-    new_ast_sources: set[str] = set()
-    new_sem_sources: set[str] = set()
-    for n in new.get("nodes", []):
-        if not isinstance(n, dict):
-            continue
-        sf = n.get("source_file")
-        if not sf:
-            continue
-        tier_sources = new_ast_sources if _is_ast_tier(n) else new_sem_sources
-        tier_sources.add(sf)
-        norm = _norm_source_file(sf, _eff_root)
-        if norm:
-            tier_sources.add(norm)
+    # semantic layer and vice versa. AST replacement ownership is derived from
+    # explicit extraction provenance (ast_sources or "extracted_sources"),
+    # falling back to AST node source_files only when no provenance is provided.
+    new_ast_sources, new_sem_sources = _tier_replacement_sources(
+        [new], root=_eff_root, ast_sources=ast_sources
+    )
     new_sources: set[str] = new_ast_sources | new_sem_sources
 
     # "Replace" wins over a contradictory "delete" of the same source (#1796),
@@ -1715,6 +1778,7 @@ def build_merge(
     dedup: bool = True,
     dedup_llm_backend: str | None = None,
     root: str | Path | None = None,
+    ast_sources: "Iterable[str | Path] | None" = None,
 ) -> nx.Graph:
     """Load existing graph.json and return it merged with ``new_chunks``.
 
@@ -1722,13 +1786,16 @@ def build_merge(
     ``export.to_json(G, communities, graph_path, force=True)`` after
     clustering. ``graph_path`` is read-only here.
 
-    Re-extracted files REPLACE their prior contribution per tier (#2333/#2336):
+    Re-extracted files REPLACE their prior contribution per tier (#2333/#2336, #3411):
     a source_file present in new_chunks has its existing nodes/edges dropped
     for each tier (AST vs semantic, per :func:`_is_ast_tier`) the new chunks
     actually contain, so a changed file's stale nodes/edges don't accumulate
-    while a one-tier re-extract keeps the other tier's layer intact. Files
-    absent from new_chunks are preserved unchanged; deleted files are removed
-    via prune_sources (tier-blind). Safe to call repeatedly.
+    while a one-tier re-extract keeps the other tier's layer intact. AST replacement
+    ownership is derived from explicit extraction provenance (``ast_sources`` or
+    chunk-level ``extracted_sources``), falling back to AST node source_file only
+    when no provenance is provided (#3411). Files absent from new_chunks are
+    preserved unchanged; deleted files are removed via prune_sources (tier-blind).
+    Safe to call repeatedly.
     root: if given, absolute source_file paths in new_chunks are made relative (#932).
     directed: if None (default), honor the on-disk graph's own ``directed`` flag
     when one exists, so an incremental merge can't silently flip a directed
@@ -1781,19 +1848,17 @@ def build_merge(
     # chunk used to delete the file's AST headings). Which tier a NEW chunk
     # item belongs to is read via _is_ast_tier (existing items were stamped by
     # _load_existing_graph above).
+    #
+    # #3411: AST replacement ownership is derived from explicit extraction
+    # provenance (ast_sources or chunk-level "extracted_sources"), so cross-file
+    # stub nodes emitted by extractors (e.g. .sln project stubs or ProjectReference
+    # stubs) do not pollute the replacement set and wipe the target project's
+    # nodes/edges. Falls back to AST node source_file only when no provenance is
+    # provided.
     _replace_root = _eff_root
-    new_ast_sources: set[str] = set()
-    new_sem_sources: set[str] = set()
-    for ch in new_chunks:
-        for n in ch.get("nodes", []):
-            sf = n.get("source_file")
-            if not sf:
-                continue
-            tier_sources = new_ast_sources if _is_ast_tier(n) else new_sem_sources
-            tier_sources.add(sf)
-            norm = _norm_source_file(sf, _replace_root)
-            if norm:
-                tier_sources.add(norm)
+    new_ast_sources, new_sem_sources = _tier_replacement_sources(
+        new_chunks, root=_replace_root, ast_sources=ast_sources
+    )
     new_sources: set[str] = new_ast_sources | new_sem_sources
     # True on-disk baseline for the #479 shrink accounting at the end (#2497):
     # the rebind below removes the re-extracted sources' old nodes from

graphify/cli.py

@@ -4091,6 +4091,7 @@ def dispatch_command(cmd: str) -> None:
             "hyperedges": list(sem_result.get("hyperedges", [])),
             "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0),
             "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0),
+            "extracted_sources": list(ast_result.get("extracted_sources", [])),
         }
 
         graph_json_path = graphify_out / "graph.json"

graphify/extract.py

@@ -7627,6 +7627,10 @@ def extract(
         # manifest does not freeze them as processed (#2543). Callers that
         # only read nodes/edges ignore this key.
         "failed_sources": _failed_sources,
+        # Surfaces the actual dispatched source paths so build_merge /
+        # merge_raw_extraction know which files were genuinely re-extracted
+        # rather than guessing ownership from node["source_file"] (#3411).
+        "extracted_sources": [str(p) for p in paths],
     }
 
 

tests/test_build.py

@@ -1713,3 +1713,273 @@ def test_derive_prune_root_recovers_root_from_posix_absolute_prune_sources():
     assert _derive_prune_root(
         ["/home/ci/build/repo/docs/a.md"], stored
     ) == "/home/ci/build/repo"
+
+
+# ── #3411: Cross-file stub nodes must not pollute AST replacement set ────────
+
+
+def test_build_merge_sln_stub_does_not_replace_referenced_csproj(tmp_path):
+    """#3411: Re-extracting a .sln must replace only the .sln's AST contribution.
+    Cross-file project stubs (source_file='A/A.csproj') in the .sln chunk must not
+    cause A/A.csproj's PackageReference/TargetFramework nodes and edges to be wiped."""
+    import networkx as nx
+
+    root = tmp_path / "corpus"
+    root.mkdir()
+    graph_path = tmp_path / "graph.json"
+
+    # Initial full extraction:
+    # 1) A.csproj with framework, package reference, and internal edges
+    # 2) A.sln containing A.csproj
+    c_proj = {
+        "nodes": [
+            {"id": "a_a_csproj", "label": "A.csproj", "file_type": "code",
+             "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"},
+            {"id": "framework_net8_0", "label": "net8.0", "file_type": "concept",
+             "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"},
+            {"id": "nuget_newtonsoft_json", "label": "Newtonsoft.Json (13.0.3)",
+             "file_type": "code", "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"},
+        ],
+        "edges": [
+            {"source": "a_a_csproj", "target": "framework_net8_0", "relation": "references",
+             "confidence": "EXTRACTED", "source_file": "A/A.csproj", "weight": 1.0, "_origin": "ast"},
+            {"source": "a_a_csproj", "target": "nuget_newtonsoft_json", "relation": "imports",
+             "confidence": "EXTRACTED", "source_file": "A/A.csproj", "weight": 1.0, "_origin": "ast"},
+        ],
+        "extracted_sources": [str(root / "A" / "A.csproj")],
+    }
+    sln_initial = {
+        "nodes": [
+            {"id": "a_sln", "label": "A.sln", "file_type": "code",
+             "source_file": "A.sln", "source_location": None, "_origin": "ast"},
+            {"id": "a_a_csproj", "label": "A", "file_type": "code",
+             "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"},
+        ],
+        "edges": [
+            {"source": "a_sln", "target": "a_a_csproj", "relation": "contains",
+             "confidence": "EXTRACTED", "source_file": "A.sln", "weight": 1.0, "_origin": "ast"},
+        ],
+        "extracted_sources": [str(root / "A.sln")],
+    }
+    G0 = build([c_proj, sln_initial], dedup=False, root=root)
+    graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8")
+
+    # Incremental update: ONLY A.sln is modified and re-extracted.
+    # extract_sln produces the solution node AND a cross-file stub for A/A.csproj.
+    sln_reextracted = {
+        "nodes": [
+            {"id": "a_sln", "label": "A.sln", "file_type": "code",
+             "source_file": "A.sln", "source_location": None, "_origin": "ast"},
+            {"id": "a_a_csproj", "label": "A", "file_type": "code",
+             "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"},
+        ],
+        "edges": [
+            {"source": "a_sln", "target": "a_a_csproj", "relation": "contains",
+             "confidence": "EXTRACTED", "source_file": "A.sln", "weight": 1.0, "_origin": "ast"},
+        ],
+        # Explicit provenance: only A.sln was actually extracted
+        "extracted_sources": [str(root / "A.sln")],
+    }
+
+    G1 = build_merge([sln_reextracted], graph_path, dedup=False, root=root)
+
+    node_ids = set(G1.nodes())
+    # 1. Solution node and project stub node are present
+    assert "a_sln" in node_ids, "Solution node must be present"
+    assert "a_a_csproj" in node_ids, "Project node must be present"
+    # 2. PackageReference and TargetFramework nodes of A.csproj must SURVIVE
+    assert "framework_net8_0" in node_ids, (
+        "#3411 regression: TargetFramework node of referenced .csproj was wiped by .sln re-extraction"
+    )
+    assert "nuget_newtonsoft_json" in node_ids, (
+        "#3411 regression: PackageReference node of referenced .csproj was wiped by .sln re-extraction"
+    )
+    # 3. Edges must SURVIVE
+    assert G1.has_edge("a_sln", "a_a_csproj"), "sln contains edge must be present"
+    assert G1.has_edge("a_a_csproj", "framework_net8_0"), (
+        "#3411 regression: references edge to TargetFramework was wiped"
+    )
+    assert G1.has_edge("a_a_csproj", "nuget_newtonsoft_json"), (
+        "#3411 regression: imports edge to PackageReference was wiped"
+    )
+
+
+def test_build_merge_project_reference_stub_does_not_replace_referenced_project(tmp_path):
+    """#3411: Re-extracting B.csproj (which has a ProjectReference to A.csproj) must
+    replace only B.csproj's AST contribution. A.csproj's PackageReference and
+    TargetFramework nodes must remain intact."""
+    import networkx as nx
+
+    root = tmp_path / "corpus"
+    root.mkdir()
+    graph_path = tmp_path / "graph.json"
+
+    # Initial graph: A.csproj with package & framework; B.csproj referencing A.csproj
+    c_proj_a = {
+        "nodes": [
+            {"id": "a_csproj", "label": "A.csproj", "file_type": "code",
+             "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"},
+            {"id": "framework_net8_0", "label": "net8.0", "file_type": "concept",
+             "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"},
+            {"id": "nuget_pkg_a", "label": "PackageA (1.0.0)", "file_type": "code",
+             "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"},
+        ],
+        "edges": [
+            {"source": "a_csproj", "target": "framework_net8_0", "relation": "references",
+             "confidence": "EXTRACTED", "source_file": "A/A.csproj", "weight": 1.0, "_origin": "ast"},
+            {"source": "a_csproj", "target": "nuget_pkg_a", "relation": "imports",
+             "confidence": "EXTRACTED", "source_file": "A/A.csproj", "weight": 1.0, "_origin": "ast"},
+        ],
+        "extracted_sources": [str(root / "A" / "A.csproj")],
+    }
+    c_proj_b = {
+        "nodes": [
+            {"id": "b_csproj", "label": "B.csproj", "file_type": "code",
+             "source_file": "B/B.csproj", "source_location": None, "_origin": "ast"},
+            {"id": "b_old_symbol", "label": "old_fn", "file_type": "code",
+             "source_file": "B/B.csproj", "source_location": "L10", "_origin": "ast"},
+            {"id": "a_csproj", "label": "A.csproj", "file_type": "code",
+             "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"},
+        ],
+        "edges": [
+            {"source": "b_csproj", "target": "b_old_symbol", "relation": "contains",
+             "confidence": "EXTRACTED", "source_file": "B/B.csproj", "weight": 1.0, "_origin": "ast"},
+            {"source": "b_csproj", "target": "a_csproj", "relation": "imports",
+             "confidence": "EXTRACTED", "source_file": "B/B.csproj", "weight": 1.0, "_origin": "ast"},
+        ],
+        "extracted_sources": [str(root / "B" / "B.csproj")],
+    }
+    G0 = build([c_proj_a, c_proj_b], dedup=False, root=root)
+    graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8")
+
+    # Re-extract ONLY B.csproj (b_old_symbol replaced by b_new_symbol; ProjectReference stub to A.csproj emitted)
+    c_proj_b_reextracted = {
+        "nodes": [
+            {"id": "b_csproj", "label": "B.csproj", "file_type": "code",
+             "source_file": "B/B.csproj", "source_location": None, "_origin": "ast"},
+            {"id": "b_new_symbol", "label": "new_fn", "file_type": "code",
+             "source_file": "B/B.csproj", "source_location": "L12", "_origin": "ast"},
+            {"id": "a_csproj", "label": "A.csproj", "file_type": "code",
+             "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"},
+        ],
+        "edges": [
+            {"source": "b_csproj", "target": "b_new_symbol", "relation": "contains",
+             "confidence": "EXTRACTED", "source_file": "B/B.csproj", "weight": 1.0, "_origin": "ast"},
+            {"source": "b_csproj", "target": "a_csproj", "relation": "imports",
+             "confidence": "EXTRACTED", "source_file": "B/B.csproj", "weight": 1.0, "_origin": "ast"},
+        ],
+        "extracted_sources": [str(root / "B" / "B.csproj")],
+    }
+
+    G1 = build_merge([c_proj_b_reextracted], graph_path, dedup=False, root=root)
+
+    node_ids = set(G1.nodes())
+    # 1. B.csproj was replaced properly
+    assert "b_new_symbol" in node_ids, "New symbol in B.csproj must be present"
+    assert "b_old_symbol" not in node_ids, "Stale symbol in B.csproj must be dropped"
+    # 2. A.csproj package & framework nodes must NOT be dropped
+    assert "nuget_pkg_a" in node_ids, (
+        "#3411 regression: PackageReference in referenced A.csproj was wiped when B.csproj was re-extracted"
+    )
+    assert "framework_net8_0" in node_ids, (
+        "#3411 regression: TargetFramework in referenced A.csproj was wiped when B.csproj was re-extracted"
+    )
+    # 3. All edges survive
+    assert G1.has_edge("a_csproj", "nuget_pkg_a")
+    assert G1.has_edge("a_csproj", "framework_net8_0")
+    assert G1.has_edge("b_csproj", "a_csproj")
+
+
+def test_merge_raw_extraction_cross_file_stub_parity(tmp_path):
+    """#3411: merge_raw_extraction shares the same explicit extraction provenance
+    scoping as build_merge."""
+    import networkx as nx
+    from graphify.build import merge_raw_extraction
+
+    root = tmp_path / "corpus"
+    root.mkdir()
+    graph_path = tmp_path / "graph.json"
+
+    # Initial graph on disk
+    c_proj = {
+        "nodes": [
+            {"id": "a_csproj", "label": "A.csproj", "file_type": "code",
+             "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"},
+            {"id": "nuget_pkg", "label": "PackageA", "file_type": "code",
+             "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"},
+        ],
+        "edges": [
+            {"source": "a_csproj", "target": "nuget_pkg", "relation": "imports",
+             "confidence": "EXTRACTED", "source_file": "A/A.csproj", "weight": 1.0, "_origin": "ast"},
+        ],
+    }
+    G0 = build([c_proj], dedup=False, root=root)
+    graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8")
+
+    # Re-extract A.sln with cross-file stub for A.csproj
+    sln_reextracted = {
+        "nodes": [
+            {"id": "a_sln", "label": "A.sln", "file_type": "code",
+             "source_file": "A.sln", "source_location": None, "_origin": "ast"},
+            {"id": "a_csproj", "label": "A", "file_type": "code",
+             "source_file": "A/A.csproj", "source_location": None, "_origin": "ast"},
+        ],
+        "edges": [
+            {"source": "a_sln", "target": "a_csproj", "relation": "contains",
+             "confidence": "EXTRACTED", "source_file": "A.sln", "weight": 1.0, "_origin": "ast"},
+        ],
+        "extracted_sources": ["A.sln"],
+    }
+
+    merged = merge_raw_extraction(sln_reextracted, graph_path=graph_path, root=root)
+    merged_node_ids = {n["id"] for n in merged["nodes"]}
+
+    assert "nuget_pkg" in merged_node_ids, (
+        "#3411: raw incremental path wiped PackageReference from referenced .csproj"
+    )
+    assert any(
+        e.get("source") == "a_csproj" and e.get("target") == "nuget_pkg"
+        for e in merged["edges"]
+    ), "#3411: raw incremental path wiped imports edge to PackageReference"
+
+
+def test_build_merge_explicit_ast_sources_argument(tmp_path):
+    """#3411: build_merge honors ast_sources argument when passed explicitly."""
+    import networkx as nx
+
+    root = tmp_path / "corpus"
+    root.mkdir()
+    graph_path = tmp_path / "graph.json"
+
+    c_proj = {
+        "nodes": [
+            {"id": "a_csproj", "label": "A.csproj", "file_type": "code",
+             "source_file": "A/A.csproj", "_origin": "ast"},
+            {"id": "nuget_pkg", "label": "PackageA", "file_type": "code",
+             "source_file": "A/A.csproj", "_origin": "ast"},
+        ],
+        "edges": [
+            {"source": "a_csproj", "target": "nuget_pkg", "relation": "imports",
+             "source_file": "A/A.csproj", "_origin": "ast"},
+        ],
+    }
+    G0 = build([c_proj], dedup=False, root=root)
+    graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8")
+
+    # Chunk has nodes without 'extracted_sources' attached to chunk,
+    # but ast_sources is provided explicitly to build_merge
+    sln_chunk = {
+        "nodes": [
+            {"id": "a_sln", "label": "A.sln", "file_type": "code",
+             "source_file": "A.sln", "_origin": "ast"},
+            {"id": "a_csproj", "label": "A", "file_type": "code",
+             "source_file": "A/A.csproj", "_origin": "ast"},
+        ],
+        "edges": [
+            {"source": "a_sln", "target": "a_csproj", "relation": "contains",
+             "source_file": "A.sln", "_origin": "ast"},
+        ],
+    }
+
+    G1 = build_merge([sln_chunk], graph_path, dedup=False, root=root, ast_sources=["A.sln"])
+    assert "nuget_pkg" in G1, "ast_sources explicit arg must protect undispatched A.csproj"