diff --git a/graphify/build.py b/graphify/build.py index 2a3b5bb8..a0821979 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -34,6 +34,23 @@ from .paths import default_graph_json as _default_graph_json from .validate import validate_extraction +# Deterministic (AST) extractors emit source_location "L"; the semantic +# extraction spec emits null. Used by _is_ast_tier as a shape fallback for +# legacy items that predate the _origin marker (#2334). +_AST_LOC_RE = re.compile(r"^L\d") + + +def _is_ast_tier(item: dict) -> bool: + """AST vs semantic tier. _origin wins when present; unstamped legacy items + (pre-0.9.16) fall back to shape: deterministic extractors emit + source_location 'L', the semantic spec emits null (#2334).""" + o = item.get("_origin") + if o is not None: + return o == "ast" + loc = item.get("source_location") + return isinstance(loc, str) and bool(_AST_LOC_RE.match(loc)) + + # Language interop families, keyed by extension, for the cross-language phantom-edge # guard in the edge loop below. Families group by REAL interop (JS/TS share a module # graph; C/C++/ObjC share a compilation unit via headers; JVM langs share bytecode), @@ -454,7 +471,7 @@ def _semantic_id_remap(nodes: list, root: str | None) -> dict: for node in nodes: if not isinstance(node, dict): continue - if node.get("_origin") == "ast": + if _is_ast_tier(node): continue nid = node.get("id") sf = node.get("source_file") @@ -781,6 +798,13 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat sf = str(attrs.get("source_file", "")) if not label or not sf: continue + # Strict _origin check on purpose — NOT _is_ast_tier (#2334): existing + # graph items are backfilled with _origin at load time, so inside a + # build the only unstamped items are fresh SEMANTIC chunks (extract() + # always stamps AST output). Those may carry drifted 'L' + # source_locations (the very ghosts #1145-extended collapses), and the + # shape fallback would misread them as AST — turning two same-file LLM + # duplicates into a fake AST/AST collision that blocks their merge. is_ast = attrs.get("_origin") == "ast" if attrs.get("source_location") or is_ast: # Key on the FULL normalized source_file, not the bare basename @@ -809,7 +833,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat for nid in sorted(node_set): attrs = G.nodes[nid] if attrs.get("_origin") == "ast": - continue # AST nodes are never ghosts + continue # AST nodes are never ghosts (strict check — see Pass 1) label = str(attrs.get("label", "")).strip() sf = str(attrs.get("source_file", "")) if not label or not sf: @@ -1216,9 +1240,23 @@ def _load_existing_graph(graph_path: Path) -> "tuple[list, list, list, bool] | N "Delete the file and run a full rebuild." ) from exc links_key = "links" if "links" in data else "edges" + nodes = list(data.get("nodes", [])) + edges = list(data.get(links_key, [])) + # Backfill tier provenance on legacy items (#2334): _origin is stamped at + # extraction time only (extract.py for AST, and the semantic path never + # stamps), so pre-0.9.16 graphs and externally-merged fragments carry + # unstamped items. Stamp them via the _is_ast_tier shape fallback so the + # graph self-heals on the next write and every downstream tier decision + # (build_merge replace, watch reconcile) reads an explicit marker. + for item in nodes: + if isinstance(item, dict): + item.setdefault("_origin", "ast" if _is_ast_tier(item) else "semantic") + for item in edges: + if isinstance(item, dict): + item.setdefault("_origin", "ast" if _is_ast_tier(item) else "semantic") return ( - list(data.get("nodes", [])), - list(data.get(links_key, [])), + nodes, + edges, list(data.get("hyperedges", [])), bool(data.get("directed", False)), ) @@ -1236,8 +1274,10 @@ def merge_raw_extraction( Replace/prune semantics mirror :func:`build_merge` exactly, so the raw and clustered incremental paths can't drift: - - sources re-extracted this run REPLACE their prior contribution — existing - nodes/edges/hyperedges owned by them are dropped, matched in both raw and + - sources re-extracted this run REPLACE their prior contribution PER TIER + (#2333/#2336): existing nodes/edges/hyperedges owned by them are dropped + only when the new extraction contains the same tier (AST vs semantic, + per :func:`_is_ast_tier`) for that source, matched in both raw and :func:`_norm_source_file` form (#1007); - ``prune_sources`` (deleted / excluded / graph-stale files) are dropped, with the ``_abs_identity`` third-form fallback (#2012), and "replace" wins @@ -1265,17 +1305,24 @@ def merge_raw_extraction( else _infer_merge_root(graph_path) ) - new_sources: set[str] = set() + # Tier-scoped replace, mirroring build_merge (#2333/#2336, COEXIST): a + # source re-extracted this run replaces only the tier(s) actually present + # in the new extraction, so an AST-only re-extract keeps the file's + # semantic layer and vice versa. + new_ast_sources: set[str] = set() + new_sem_sources: set[str] = set() for n in new.get("nodes", []): if not isinstance(n, dict): continue sf = n.get("source_file") if not sf: continue - new_sources.add(sf) + tier_sources = new_ast_sources if _is_ast_tier(n) else new_sem_sources + tier_sources.add(sf) norm = _norm_source_file(sf, _eff_root) if norm: - new_sources.add(norm) + tier_sources.add(norm) + new_sources: set[str] = new_ast_sources | new_sem_sources prune_set: set[str] = set() prune_abs: set[str] = set() @@ -1300,7 +1347,12 @@ def merge_raw_extraction( if not isinstance(item, dict): return True sf = item.get("source_file") - if sf in new_sources or _norm_source_file(sf, _eff_root) in new_sources: + # Tier-scoped replace: an item is superseded only when ITS OWN tier + # re-extracted its source. Hyperedges are semantic-tier (no _origin, + # null source_location), so an AST-only re-extract carries them. + # Deletion pruning below stays tier-blind. + own = new_ast_sources if _is_ast_tier(item) else new_sem_sources + if sf in own or _norm_source_file(sf, _eff_root) in own: return True # re-extracted this run — replaced by the new chunk if not sf: return False # unowned — carry forward @@ -1332,11 +1384,13 @@ def build_merge( ) -> nx.Graph: """Load existing graph.json, merge new chunks into it, and save back. - Re-extracted files REPLACE their prior contribution: any source_file present - in new_chunks is dropped from the loaded graph before merging, so a changed - file's stale nodes/edges don't accumulate. Files absent from new_chunks are - preserved unchanged; deleted files are removed via prune_sources. - Safe to call repeatedly. + Re-extracted files REPLACE their prior contribution per tier (#2333/#2336): + a source_file present in new_chunks has its existing nodes/edges dropped + for each tier (AST vs semantic, per :func:`_is_ast_tier`) the new chunks + actually contain, so a changed file's stale nodes/edges don't accumulate + while a one-tier re-extract keeps the other tier's layer intact. Files + absent from new_chunks are preserved unchanged; deleted files are removed + via prune_sources (tier-blind). Safe to call repeatedly. root: if given, absolute source_file paths in new_chunks are made relative (#932). directed: if None (default), honor the on-disk graph's own ``directed`` flag when one exists, so an incremental merge can't silently flip a directed @@ -1379,21 +1433,32 @@ def build_merge( # for them; genuinely deleted files are still handled via prune_sources. # Matched in both raw and _norm_source_file form because new_chunks may carry # absolute win32 paths while the stored graph keeps relative posix (#1007). + # Replacement is tier-scoped (#2333/#2336, COEXIST): each file has two + # producers — the deterministic AST pass and the semantic/LLM pass — whose + # node sets coexist in the graph. A re-extract of one tier must replace + # only that tier's prior contribution, never the other's (a semantic-only + # chunk used to delete the file's AST headings). Which tier a NEW chunk + # item belongs to is read via _is_ast_tier (existing items were stamped by + # _load_existing_graph above). _replace_root = _eff_root - new_sources: set[str] = set() + new_ast_sources: set[str] = set() + new_sem_sources: set[str] = set() for ch in new_chunks: for n in ch.get("nodes", []): sf = n.get("source_file") if not sf: continue - new_sources.add(sf) + tier_sources = new_ast_sources if _is_ast_tier(n) else new_sem_sources + tier_sources.add(sf) norm = _norm_source_file(sf, _replace_root) if norm: - new_sources.add(norm) + tier_sources.add(norm) + new_sources: set[str] = new_ast_sources | new_sem_sources if new_sources: def _kept(item: dict) -> bool: sf = item.get("source_file") - return sf not in new_sources and _norm_source_file(sf, _replace_root) not in new_sources + own = new_ast_sources if _is_ast_tier(item) else new_sem_sources + return sf not in own and _norm_source_file(sf, _replace_root) not in own existing_nodes = [n for n in existing_nodes if _kept(n)] existing_edges = [e for e in existing_edges if _kept(e)] @@ -1461,8 +1526,11 @@ def build_merge( continue sf = he.get("source_file") norm = _norm_source_file(sf, _eff_root) - if sf in new_sources or norm in new_sources: - continue # re-extracted — replaced by the new chunk's version + # Hyperedges are semantic-tier: only a SEMANTIC re-extract of the + # source replaces them. An AST-only re-extract cannot regenerate + # hyperedges, so dropping them there would be data loss (#2336). + if sf in new_sem_sources or norm in new_sem_sources: + continue # semantically re-extracted — replaced by the new chunk's version if _prune_match(sf): continue # deleted — pruned carried.append(he) diff --git a/graphify/watch.py b/graphify/watch.py index ff556f0b..a1adb64b 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -491,6 +491,16 @@ def _reconcile_existing_graph( existing = json.loads(existing_graph.read_text(encoding="utf-8")) existing_graph_data = existing + # Backfill tier provenance on legacy items (#2334), mirroring + # build._load_existing_graph (this reconcile path loads the raw dict + # separately, so the backfill there does not reach it). Stamping preserved + # items means the graph self-heals on this write. + from graphify.build import _is_ast_tier + for _bucket in ("nodes", "links", "edges"): + for _item in existing.get(_bucket, []): + if isinstance(_item, dict): + _item.setdefault("_origin", "ast" if _is_ast_tier(_item) else "semantic") + try: from graphify.build import _norm_source_file as _nsf from graphify.extract import _get_extractor @@ -584,15 +594,20 @@ def _reconcile_existing_graph( "Run a full re-extraction to purge them if the exclusion is intentional." ) - # A full re-extraction owns every AST node under watch_root. Incremental - # extraction owns only nodes from rebuilt or deleted sources. Semantic - # nodes lack the AST origin marker and remain preserved. + # A full re-extraction owns the AST nodes of every source it actually + # re-extracted (extract_targets, via rebuilt_source_identities) — NOT + # every AST node under watch_root: a semantic-backed doc is excluded + # from extract_targets (#1915), so its existing AST layer is not + # regenerated this run and dropping it would be data loss (#2333, + # COEXIST — the AST and semantic layers of a file coexist). + # Incremental extraction owns only nodes from rebuilt or deleted + # sources. Semantic-tier nodes (per _is_ast_tier) remain preserved. preserved_nodes = [ node for node in existing.get("nodes", []) if node["id"] not in new_ast_ids and not ( - node.get("_origin") == "ast" + _is_ast_tier(node) and ( ( not node.get("source_file") @@ -600,7 +615,7 @@ def _reconcile_existing_graph( ) or ( full_rebuild - and source_paths.in_watch_root(node.get("source_file")) + and source_paths.is_evicted(node, rebuilt_source_identities) ) ) ) @@ -621,7 +636,7 @@ def _reconcile_existing_graph( and edge.get("target") in all_ids and not source_paths.is_evicted(edge, edge_evicted_source_identities) and not ( - edge.get("_origin") == "ast" + _is_ast_tier(edge) and source_paths.is_evicted(edge, rebuilt_source_identities) ) ] @@ -973,7 +988,7 @@ def _rebuild_code( try: from graphify.extract import extract, _get_extractor from graphify.detect import detect - from graphify.build import build_from_json, _norm_source_file as _nsf + from graphify.build import build_from_json, _is_ast_tier, _norm_source_file as _nsf from graphify.cluster import cluster, remap_communities_to_previous, score_all from graphify.analyze import god_nodes, surprising_connections, suggest_questions from graphify.report import generate @@ -1008,10 +1023,12 @@ def _rebuild_code( # existing graph must not ALSO be AST-quick-scanned — otherwise every # rebuild mints heading nodes on top of the preserved semantic nodes # and the doc is represented twice (~4x graph bloat vs the CLI update - # path, which AST-extracts only code). Semantic supersedes AST per doc - # source: the quick-scan stays as a fallback for docs with no semantic - # layer (the no-LLM doc-structure feature, #09b33b7) and for brand-new - # docs the graph has never seen. These docs stay in ``code_files`` so + # path, which AST-extracts only code). A semantic-backed doc is never + # re-quick-scanned, and any AST layer it already carries coexists and + # is preserved rather than regenerated (#2333, COEXIST); the + # quick-scan stays as a fallback for docs with no semantic layer (the + # no-LLM doc-structure feature, #09b33b7) and for brand-new docs the + # graph has never seen. These docs stay in ``code_files`` so # corpus membership (#1795 fail-closed deletion evidence) and the # shrink accounting below still cover them — a previously-bloated # graph must be allowed to self-heal on a full rebuild without the @@ -1044,7 +1061,11 @@ def _rebuild_code( # "document" nodes (extractors/markdown.py). "image" stays out. semantic_doc_identities: set[str] = set() for node in prior.get("nodes", []): - if node.get("_origin") == "ast": + # _is_ast_tier, not a raw _origin check (#2334): a legacy + # unstamped AST heading node (source_location "L") must + # not fake a semantic layer, or the doc would be excluded + # from the AST quick-scan forever. + if _is_ast_tier(node): continue if node.get("file_type") not in ( "document", "concept", "rationale", "paper", "code" @@ -1129,10 +1150,11 @@ def _rebuild_code( extract_targets = wanted else: # Full rebuild: skip the AST quick-scan for semantic-backed docs - # (#1915). They remain in code_files, so stale _origin=="ast" - # heading nodes from a previously-bloated graph are dropped by the - # full-rebuild AST ownership rule while the shrink accounting - # below still counts the doc as a rebuilt source. + # (#1915). They remain in code_files for corpus membership and + # shrink accounting, but because they are not extract targets the + # full-rebuild AST ownership rule (scoped to + # rebuilt_source_identities, #2333 COEXIST) leaves their existing + # AST heading layer intact alongside the semantic layer. extract_targets = [p for p in code_files if p not in semantic_doc_files] commit = _git_head(cwd=watch_root) diff --git a/tests/test_build.py b/tests/test_build.py index 098a2b38..e51da3ff 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -1046,6 +1046,142 @@ def test_build_merge_replaces_changed_file_stale_edges(tmp_path): assert ("K", "A") in edges, "unchanged file's edge must survive" +def _write_two_tier_graph(graph_path): + """A graph where docs/readme.md carries BOTH tiers (#2333 COEXIST): an + AST layer (document/heading nodes + a contains edge, _origin=ast) and a + semantic layer (an unstamped concept node, source_location=None).""" + data = { + "directed": False, + "nodes": [ + {"id": "docs_readme", "label": "Readme", "file_type": "document", + "source_file": "docs/readme.md", "source_location": "L1", + "_origin": "ast"}, + {"id": "docs_readme_intro", "label": "Intro", "file_type": "document", + "source_file": "docs/readme.md", "source_location": "L3", + "_origin": "ast"}, + {"id": "auth_flow", "label": "Auth Flow", "file_type": "concept", + "source_file": "docs/readme.md", "source_location": None}, + ], + "links": [ + {"source": "docs_readme", "target": "docs_readme_intro", + "relation": "contains", "confidence": "EXTRACTED", + "source_file": "docs/readme.md", "source_location": "L3", + "weight": 1.0, "_origin": "ast"}, + ], + "hyperedges": [ + {"id": "auth_group", "label": "Auth Group", + "nodes": ["docs_readme", "auth_flow"], "relation": "form", + "confidence": "INFERRED", "source_file": "docs/readme.md"}, + ], + } + graph_path.write_text(json.dumps(data), encoding="utf-8") + + +def test_build_merge_semantic_reextract_preserves_ast_layer(tmp_path): + """#2333/#2336 (COEXIST): a semantic-only re-extract of a file replaces + only that file's SEMANTIC tier — its AST document/heading nodes and AST + edges must survive. Before the fix, replace-by-source was tier-blind and + the semantic chunk deleted the doc's AST layer.""" + graph_path = tmp_path / "graph.json" + _write_two_tier_graph(graph_path) + + # Semantic-only chunk for the same file: no _origin, null source_location + # (the extraction-spec shape — the semantic path never stamps _origin). + chunk = {"nodes": [ + {"id": "session_model", "label": "Session Model", "file_type": "concept", + "source_file": "docs/readme.md", "source_location": None}, + ], "edges": []} + G = build_merge([chunk], graph_path, dedup=False) + + assert "docs_readme" in G and "docs_readme_intro" in G, ( + "AST layer deleted by a semantic-only re-extract of the same file" + ) + assert G.has_edge("docs_readme", "docs_readme_intro"), ( + "AST contains edge deleted by a semantic-only re-extract" + ) + assert "auth_flow" not in G, "old semantic node must be replaced" + assert "session_model" in G, "new semantic node must be present" + + +def test_build_merge_ast_reextract_preserves_semantic_layer(tmp_path): + """#2333/#2336 inverse: an AST-only re-extract of a file replaces only + that file's AST tier — its semantic concept nodes (and semantic-tier + hyperedges, which an AST pass cannot regenerate) must survive.""" + graph_path = tmp_path / "graph.json" + _write_two_tier_graph(graph_path) + + # AST-only re-extract: the doc changed — Intro heading replaced by + # Quickstart. extract() always stamps _origin=ast. + chunk = {"nodes": [ + {"id": "docs_readme", "label": "Readme", "file_type": "document", + "source_file": "docs/readme.md", "source_location": "L1", + "_origin": "ast"}, + {"id": "docs_readme_quickstart", "label": "Quickstart", + "file_type": "document", "source_file": "docs/readme.md", + "source_location": "L5", "_origin": "ast"}, + ], "edges": [ + {"source": "docs_readme", "target": "docs_readme_quickstart", + "relation": "contains", "confidence": "EXTRACTED", + "source_file": "docs/readme.md", "source_location": "L5", + "weight": 1.0, "_origin": "ast"}, + ]} + G = build_merge([chunk], graph_path, dedup=False) + + assert "auth_flow" in G, ( + "semantic layer deleted by an AST-only re-extract of the same file" + ) + assert "docs_readme_intro" not in G, "stale AST node must be replaced" + assert "docs_readme_quickstart" in G, "fresh AST node must be present" + assert G.has_edge("docs_readme", "docs_readme_quickstart") + carried_he_ids = {he.get("id") for he in G.graph.get("hyperedges", [])} + assert "auth_group" in carried_he_ids, ( + "semantic-tier hyperedge dropped by an AST-only re-extract (#2336)" + ) + + +def test_merge_raw_extraction_tier_scoped(tmp_path): + """#2333 raw-path mirror: merge_raw_extraction (extract --no-cluster + incremental) applies the same tier-scoped replace as build_merge — a + semantic-only re-extract keeps the file's AST nodes/edges.""" + from graphify.build import merge_raw_extraction + + graph_path = tmp_path / "graph.json" + _write_two_tier_graph(graph_path) + + new = {"nodes": [ + {"id": "session_model", "label": "Session Model", "file_type": "concept", + "source_file": "docs/readme.md", "source_location": None}, + ], "edges": [], "input_tokens": 0, "output_tokens": 0} + out = merge_raw_extraction(new, graph_path) + + node_ids = {n["id"] for n in out["nodes"]} + assert {"docs_readme", "docs_readme_intro", "session_model"} <= node_ids, ( + "AST layer dropped by a semantic-only raw re-extract" + ) + assert "auth_flow" not in node_ids, "old semantic node must be replaced" + edge_keys = {(e.get("source"), e.get("target")) for e in out["edges"]} + assert ("docs_readme", "docs_readme_intro") in edge_keys, ( + "AST edge dropped by a semantic-only raw re-extract" + ) + he_ids = {he.get("id") for he in out.get("hyperedges", [])} + assert "auth_group" not in he_ids, ( + "semantic-tier hyperedge must be replaced by a semantic re-extract" + ) + + +def test_is_ast_tier_legacy_fallback(): + """#2334: _origin wins when present; unstamped legacy items fall back to + the source_location shape (AST emits 'L', semantic emits null).""" + from graphify.build import _is_ast_tier + + assert _is_ast_tier({"_origin": "ast"}) is True + assert _is_ast_tier({"_origin": "ast", "source_location": None}) is True + assert _is_ast_tier({"source_location": "L10"}) is True + assert _is_ast_tier({"source_location": None}) is False + assert _is_ast_tier({}) is False + assert _is_ast_tier({"_origin": "semantic", "source_location": "L10"}) is False + + def test_build_merge_root_collapses_convention_drift(tmp_path): """Skill contract: the extraction subagent must emit source_file as the verbatim path from FILE_LIST AND the caller must pass root= (the build root). diff --git a/tests/test_watch.py b/tests/test_watch.py index b33c98d3..c3029fe1 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -2050,11 +2050,13 @@ def test_rebuild_code_quick_scans_doc_without_semantic_nodes(tmp_path): assert {"notes", "notes_alpha", "notes_beta"} <= ids -def test_rebuild_code_polluted_graph_self_heals_on_full_rebuild(tmp_path): - """#1915: a graph already bloated by the bug (semantic doc nodes PLUS stale - _origin=="ast" heading nodes for the same doc) sheds the heading nodes on - the next full rebuild via the AST ownership rule — and the shrink guard - accepts the smaller write without --force.""" +def test_full_rebuild_preserves_semantic_backed_doc_ast_layer(tmp_path): + """#2333 (COEXIST): a doc that carries BOTH an AST heading layer and a + semantic layer keeps both across a full rebuild. The doc is excluded from + extract_targets (#1915, no re-quick-scan), so its AST nodes are NOT + regenerated this run — the full-rebuild ownership rule must therefore not + drop them (it owns only rebuilt sources, not everything in watch_root). + Supersedes the pre-COEXIST #1915 self-heal, which deleted the AST layer.""" from graphify.watch import _rebuild_code corpus = tmp_path / "corpus" @@ -2070,31 +2072,107 @@ def test_rebuild_code_polluted_graph_self_heals_on_full_rebuild(tmp_path): graph_path = corpus / "graphify-out" / "graph.json" data = json.loads(graph_path.read_text(encoding="utf-8")) assert _AST_GUIDE_IDS <= {n["id"] for n in data["nodes"]} + doc_nodes_before = sum( + 1 for n in data["nodes"] if n.get("source_file") == "guide.md" + ) - # Layer the semantic representation on top -> the double-represented state. - data["nodes"].extend([ - {"id": "guide_doc", "label": "Guide", "file_type": "document", - "source_file": "guide.md"}, + # Layer the semantic representation on top — both tiers now coexist. + data["nodes"].append( {"id": "auth_flow", "label": "Auth Flow", "file_type": "concept", "source_file": "guide.md"}, - ]) - data["links"].append({ - "source": "guide_doc", "target": "auth_flow", "relation": "explains", - "confidence": "INFERRED", "source_file": "guide.md", - }) + ) graph_path.write_text(json.dumps(data), encoding="utf-8") - nodes_before = len(data["nodes"]) - # No force=True: the self-heal shrink must be accepted by the guard. + # Full rebuild WITHOUT force: guide.md is now semantic-backed, so it is + # excluded from extract_targets — its existing AST layer must survive. assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True after = json.loads(graph_path.read_text(encoding="utf-8")) after_ids = {n["id"] for n in after["nodes"]} - assert {"guide_doc", "auth_flow"} <= after_ids - assert not (_AST_GUIDE_IDS & after_ids), ( - "stale AST heading nodes for a semantic-backed doc must self-heal away" + assert "auth_flow" in after_ids, "semantic layer lost on full rebuild" + assert _AST_GUIDE_IDS <= after_ids, ( + "AST heading layer of a semantic-backed doc dropped by a full " + "rebuild (#2333 COEXIST)" + ) + doc_nodes_after = sum( + 1 for n in after["nodes"] + if n.get("source_file") == "guide.md" and n["id"] != "auth_flow" + ) + assert doc_nodes_after == doc_nodes_before, ( + "document-node count changed across a full rebuild of a " + f"semantic-backed doc: {doc_nodes_before} -> {doc_nodes_after}" + ) + + +def test_full_rebuild_regenerates_docs_with_legacy_unstamped_nodes(tmp_path): + """#2334: a legacy heading node without the _origin stamp (pre-0.9.16 + graph) must not fake a semantic layer — _is_ast_tier's shape fallback + (source_location "L") classifies it as AST, so the doc stays in + extract_targets, is re-quick-scanned, and comes back fully re-stamped.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text( + "def handle_login():\n return 1\n", encoding="utf-8" + ) + (corpus / "guide.md").write_text( + "# Overview\n\n## Setup\n\n## Usage\n", encoding="utf-8" + ) + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + assert _AST_GUIDE_IDS <= {n["id"] for n in data["nodes"]} + + # Simulate a legacy graph: strip the _origin stamp from one heading node. + stripped = next( + n for n in data["nodes"] if n["id"] == "guide_overview" + ) + stripped.pop("_origin", None) + graph_path.write_text(json.dumps(data), encoding="utf-8") + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + after_by_id = {n["id"]: n for n in after["nodes"]} + assert _AST_GUIDE_IDS <= set(after_by_id), ( + "legacy unstamped heading node made the doc look semantic-backed " + "and its AST structure was dropped (#2334)" + ) + for nid in _AST_GUIDE_IDS: + assert after_by_id[nid].get("_origin") == "ast", ( + f"{nid} not re-stamped with _origin=ast after the full rebuild" + ) + + +def test_full_rebuild_drops_stale_ast_for_reextracted_code(tmp_path): + """#1116 guard: tier-scoping the full-rebuild ownership rule (#2333) must + not stop a genuinely re-extracted code file from shedding its stale AST + nodes — a renamed function's old symbol node still disappears.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text( + "def old_name():\n return 1\n", encoding="utf-8" + ) + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert "app_old_name" in ids + + (corpus / "app.py").write_text( + "def new_name():\n return 1\n", encoding="utf-8" + ) + # Full rebuild (no changed_paths): app.py is re-extracted, so its stale + # AST symbol node is owned by this run and must be dropped. + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert "app_new_name" in ids + assert "app_old_name" not in ids, ( + "stale AST node for a re-extracted code file survived (#1116)" ) - assert len(after["nodes"]) < nodes_before, "polluted graph should shrink" # ── #2014: code-typed semantic nodes count as a doc's semantic layer ───────────