fix(build,watch): tier-aware merge so re-extract keeps the other layer (#2333, #2334, #2336)

Node/edge ownership was keyed on source_file alone, but each file has an AST
tier and a semantic tier. A semantic-only re-extract deleted a doc's AST
headings, and a full _rebuild_code deleted document AST nodes. Add an
_is_ast_tier() predicate (with a source_location legacy fallback for the
unreliable _origin marker, #2334), backfill _origin on load so graphs
self-heal, make build_merge / merge_raw_extraction replace by (source_file,
tier) instead of source_file (coexist policy: an AST re-extract replaces only
AST nodes and keeps the semantic layer, and vice versa), and scope the
full-rebuild AST drop to sources actually regenerated (#2336) so a
semantic-backed doc keeps its AST layer. Deletion prune stays tier-blind.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
safishamsi
2026-08-01 12:21:04 +01:00
co-authored by Claude Opus 4.8
parent c262478792
commit 97656fae2d
4 changed files with 361 additions and 57 deletions
+89 -21
View File
@@ -34,6 +34,23 @@ from .paths import default_graph_json as _default_graph_json
from .validate import validate_extraction
# Deterministic (AST) extractors emit source_location "L<line>"; the semantic
# extraction spec emits null. Used by _is_ast_tier as a shape fallback for
# legacy items that predate the _origin marker (#2334).
_AST_LOC_RE = re.compile(r"^L\d")
def _is_ast_tier(item: dict) -> bool:
"""AST vs semantic tier. _origin wins when present; unstamped legacy items
(pre-0.9.16) fall back to shape: deterministic extractors emit
source_location 'L<line>', the semantic spec emits null (#2334)."""
o = item.get("_origin")
if o is not None:
return o == "ast"
loc = item.get("source_location")
return isinstance(loc, str) and bool(_AST_LOC_RE.match(loc))
# Language interop families, keyed by extension, for the cross-language phantom-edge
# guard in the edge loop below. Families group by REAL interop (JS/TS share a module
# graph; C/C++/ObjC share a compilation unit via headers; JVM langs share bytecode),
@@ -454,7 +471,7 @@ def _semantic_id_remap(nodes: list, root: str | None) -> dict:
for node in nodes:
if not isinstance(node, dict):
continue
if node.get("_origin") == "ast":
if _is_ast_tier(node):
continue
nid = node.get("id")
sf = node.get("source_file")
@@ -781,6 +798,13 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat
sf = str(attrs.get("source_file", ""))
if not label or not sf:
continue
# Strict _origin check on purpose — NOT _is_ast_tier (#2334): existing
# graph items are backfilled with _origin at load time, so inside a
# build the only unstamped items are fresh SEMANTIC chunks (extract()
# always stamps AST output). Those may carry drifted 'L<line>'
# source_locations (the very ghosts #1145-extended collapses), and the
# shape fallback would misread them as AST — turning two same-file LLM
# duplicates into a fake AST/AST collision that blocks their merge.
is_ast = attrs.get("_origin") == "ast"
if attrs.get("source_location") or is_ast:
# Key on the FULL normalized source_file, not the bare basename
@@ -809,7 +833,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat
for nid in sorted(node_set):
attrs = G.nodes[nid]
if attrs.get("_origin") == "ast":
continue # AST nodes are never ghosts
continue # AST nodes are never ghosts (strict check — see Pass 1)
label = str(attrs.get("label", "")).strip()
sf = str(attrs.get("source_file", ""))
if not label or not sf:
@@ -1216,9 +1240,23 @@ def _load_existing_graph(graph_path: Path) -> "tuple[list, list, list, bool] | N
"Delete the file and run a full rebuild."
) from exc
links_key = "links" if "links" in data else "edges"
nodes = list(data.get("nodes", []))
edges = list(data.get(links_key, []))
# Backfill tier provenance on legacy items (#2334): _origin is stamped at
# extraction time only (extract.py for AST, and the semantic path never
# stamps), so pre-0.9.16 graphs and externally-merged fragments carry
# unstamped items. Stamp them via the _is_ast_tier shape fallback so the
# graph self-heals on the next write and every downstream tier decision
# (build_merge replace, watch reconcile) reads an explicit marker.
for item in nodes:
if isinstance(item, dict):
item.setdefault("_origin", "ast" if _is_ast_tier(item) else "semantic")
for item in edges:
if isinstance(item, dict):
item.setdefault("_origin", "ast" if _is_ast_tier(item) else "semantic")
return (
list(data.get("nodes", [])),
list(data.get(links_key, [])),
nodes,
edges,
list(data.get("hyperedges", [])),
bool(data.get("directed", False)),
)
@@ -1236,8 +1274,10 @@ def merge_raw_extraction(
Replace/prune semantics mirror :func:`build_merge` exactly, so the raw and
clustered incremental paths can't drift:
- sources re-extracted this run REPLACE their prior contribution — existing
nodes/edges/hyperedges owned by them are dropped, matched in both raw and
- sources re-extracted this run REPLACE their prior contribution PER TIER
(#2333/#2336): existing nodes/edges/hyperedges owned by them are dropped
only when the new extraction contains the same tier (AST vs semantic,
per :func:`_is_ast_tier`) for that source, matched in both raw and
:func:`_norm_source_file` form (#1007);
- ``prune_sources`` (deleted / excluded / graph-stale files) are dropped,
with the ``_abs_identity`` third-form fallback (#2012), and "replace" wins
@@ -1265,17 +1305,24 @@ def merge_raw_extraction(
else _infer_merge_root(graph_path)
)
new_sources: set[str] = set()
# Tier-scoped replace, mirroring build_merge (#2333/#2336, COEXIST): a
# source re-extracted this run replaces only the tier(s) actually present
# in the new extraction, so an AST-only re-extract keeps the file's
# semantic layer and vice versa.
new_ast_sources: set[str] = set()
new_sem_sources: set[str] = set()
for n in new.get("nodes", []):
if not isinstance(n, dict):
continue
sf = n.get("source_file")
if not sf:
continue
new_sources.add(sf)
tier_sources = new_ast_sources if _is_ast_tier(n) else new_sem_sources
tier_sources.add(sf)
norm = _norm_source_file(sf, _eff_root)
if norm:
new_sources.add(norm)
tier_sources.add(norm)
new_sources: set[str] = new_ast_sources | new_sem_sources
prune_set: set[str] = set()
prune_abs: set[str] = set()
@@ -1300,7 +1347,12 @@ def merge_raw_extraction(
if not isinstance(item, dict):
return True
sf = item.get("source_file")
if sf in new_sources or _norm_source_file(sf, _eff_root) in new_sources:
# Tier-scoped replace: an item is superseded only when ITS OWN tier
# re-extracted its source. Hyperedges are semantic-tier (no _origin,
# null source_location), so an AST-only re-extract carries them.
# Deletion pruning below stays tier-blind.
own = new_ast_sources if _is_ast_tier(item) else new_sem_sources
if sf in own or _norm_source_file(sf, _eff_root) in own:
return True # re-extracted this run — replaced by the new chunk
if not sf:
return False # unowned — carry forward
@@ -1332,11 +1384,13 @@ def build_merge(
) -> nx.Graph:
"""Load existing graph.json, merge new chunks into it, and save back.
Re-extracted files REPLACE their prior contribution: any source_file present
in new_chunks is dropped from the loaded graph before merging, so a changed
file's stale nodes/edges don't accumulate. Files absent from new_chunks are
preserved unchanged; deleted files are removed via prune_sources.
Safe to call repeatedly.
Re-extracted files REPLACE their prior contribution per tier (#2333/#2336):
a source_file present in new_chunks has its existing nodes/edges dropped
for each tier (AST vs semantic, per :func:`_is_ast_tier`) the new chunks
actually contain, so a changed file's stale nodes/edges don't accumulate
while a one-tier re-extract keeps the other tier's layer intact. Files
absent from new_chunks are preserved unchanged; deleted files are removed
via prune_sources (tier-blind). Safe to call repeatedly.
root: if given, absolute source_file paths in new_chunks are made relative (#932).
directed: if None (default), honor the on-disk graph's own ``directed`` flag
when one exists, so an incremental merge can't silently flip a directed
@@ -1379,21 +1433,32 @@ def build_merge(
# for them; genuinely deleted files are still handled via prune_sources.
# Matched in both raw and _norm_source_file form because new_chunks may carry
# absolute win32 paths while the stored graph keeps relative posix (#1007).
# Replacement is tier-scoped (#2333/#2336, COEXIST): each file has two
# producers — the deterministic AST pass and the semantic/LLM pass — whose
# node sets coexist in the graph. A re-extract of one tier must replace
# only that tier's prior contribution, never the other's (a semantic-only
# chunk used to delete the file's AST headings). Which tier a NEW chunk
# item belongs to is read via _is_ast_tier (existing items were stamped by
# _load_existing_graph above).
_replace_root = _eff_root
new_sources: set[str] = set()
new_ast_sources: set[str] = set()
new_sem_sources: set[str] = set()
for ch in new_chunks:
for n in ch.get("nodes", []):
sf = n.get("source_file")
if not sf:
continue
new_sources.add(sf)
tier_sources = new_ast_sources if _is_ast_tier(n) else new_sem_sources
tier_sources.add(sf)
norm = _norm_source_file(sf, _replace_root)
if norm:
new_sources.add(norm)
tier_sources.add(norm)
new_sources: set[str] = new_ast_sources | new_sem_sources
if new_sources:
def _kept(item: dict) -> bool:
sf = item.get("source_file")
return sf not in new_sources and _norm_source_file(sf, _replace_root) not in new_sources
own = new_ast_sources if _is_ast_tier(item) else new_sem_sources
return sf not in own and _norm_source_file(sf, _replace_root) not in own
existing_nodes = [n for n in existing_nodes if _kept(n)]
existing_edges = [e for e in existing_edges if _kept(e)]
@@ -1461,8 +1526,11 @@ def build_merge(
continue
sf = he.get("source_file")
norm = _norm_source_file(sf, _eff_root)
if sf in new_sources or norm in new_sources:
continue # re-extracted — replaced by the new chunk's version
# Hyperedges are semantic-tier: only a SEMANTIC re-extract of the
# source replaces them. An AST-only re-extract cannot regenerate
# hyperedges, so dropping them there would be data loss (#2336).
if sf in new_sem_sources or norm in new_sem_sources:
continue # semantically re-extracted — replaced by the new chunk's version
if _prune_match(sf):
continue # deleted — pruned
carried.append(he)
+38 -16
View File
@@ -491,6 +491,16 @@ def _reconcile_existing_graph(
existing = json.loads(existing_graph.read_text(encoding="utf-8"))
existing_graph_data = existing
# Backfill tier provenance on legacy items (#2334), mirroring
# build._load_existing_graph (this reconcile path loads the raw dict
# separately, so the backfill there does not reach it). Stamping preserved
# items means the graph self-heals on this write.
from graphify.build import _is_ast_tier
for _bucket in ("nodes", "links", "edges"):
for _item in existing.get(_bucket, []):
if isinstance(_item, dict):
_item.setdefault("_origin", "ast" if _is_ast_tier(_item) else "semantic")
try:
from graphify.build import _norm_source_file as _nsf
from graphify.extract import _get_extractor
@@ -584,15 +594,20 @@ def _reconcile_existing_graph(
"Run a full re-extraction to purge them if the exclusion is intentional."
)
# A full re-extraction owns every AST node under watch_root. Incremental
# extraction owns only nodes from rebuilt or deleted sources. Semantic
# nodes lack the AST origin marker and remain preserved.
# A full re-extraction owns the AST nodes of every source it actually
# re-extracted (extract_targets, via rebuilt_source_identities) — NOT
# every AST node under watch_root: a semantic-backed doc is excluded
# from extract_targets (#1915), so its existing AST layer is not
# regenerated this run and dropping it would be data loss (#2333,
# COEXIST — the AST and semantic layers of a file coexist).
# Incremental extraction owns only nodes from rebuilt or deleted
# sources. Semantic-tier nodes (per _is_ast_tier) remain preserved.
preserved_nodes = [
node
for node in existing.get("nodes", [])
if node["id"] not in new_ast_ids
and not (
node.get("_origin") == "ast"
_is_ast_tier(node)
and (
(
not node.get("source_file")
@@ -600,7 +615,7 @@ def _reconcile_existing_graph(
)
or (
full_rebuild
and source_paths.in_watch_root(node.get("source_file"))
and source_paths.is_evicted(node, rebuilt_source_identities)
)
)
)
@@ -621,7 +636,7 @@ def _reconcile_existing_graph(
and edge.get("target") in all_ids
and not source_paths.is_evicted(edge, edge_evicted_source_identities)
and not (
edge.get("_origin") == "ast"
_is_ast_tier(edge)
and source_paths.is_evicted(edge, rebuilt_source_identities)
)
]
@@ -973,7 +988,7 @@ def _rebuild_code(
try:
from graphify.extract import extract, _get_extractor
from graphify.detect import detect
from graphify.build import build_from_json, _norm_source_file as _nsf
from graphify.build import build_from_json, _is_ast_tier, _norm_source_file as _nsf
from graphify.cluster import cluster, remap_communities_to_previous, score_all
from graphify.analyze import god_nodes, surprising_connections, suggest_questions
from graphify.report import generate
@@ -1008,10 +1023,12 @@ def _rebuild_code(
# existing graph must not ALSO be AST-quick-scanned — otherwise every
# rebuild mints heading nodes on top of the preserved semantic nodes
# and the doc is represented twice (~4x graph bloat vs the CLI update
# path, which AST-extracts only code). Semantic supersedes AST per doc
# source: the quick-scan stays as a fallback for docs with no semantic
# layer (the no-LLM doc-structure feature, #09b33b7) and for brand-new
# docs the graph has never seen. These docs stay in ``code_files`` so
# path, which AST-extracts only code). A semantic-backed doc is never
# re-quick-scanned, and any AST layer it already carries coexists and
# is preserved rather than regenerated (#2333, COEXIST); the
# quick-scan stays as a fallback for docs with no semantic layer (the
# no-LLM doc-structure feature, #09b33b7) and for brand-new docs the
# graph has never seen. These docs stay in ``code_files`` so
# corpus membership (#1795 fail-closed deletion evidence) and the
# shrink accounting below still cover them — a previously-bloated
# graph must be allowed to self-heal on a full rebuild without the
@@ -1044,7 +1061,11 @@ def _rebuild_code(
# "document" nodes (extractors/markdown.py). "image" stays out.
semantic_doc_identities: set[str] = set()
for node in prior.get("nodes", []):
if node.get("_origin") == "ast":
# _is_ast_tier, not a raw _origin check (#2334): a legacy
# unstamped AST heading node (source_location "L<n>") must
# not fake a semantic layer, or the doc would be excluded
# from the AST quick-scan forever.
if _is_ast_tier(node):
continue
if node.get("file_type") not in (
"document", "concept", "rationale", "paper", "code"
@@ -1129,10 +1150,11 @@ def _rebuild_code(
extract_targets = wanted
else:
# Full rebuild: skip the AST quick-scan for semantic-backed docs
# (#1915). They remain in code_files, so stale _origin=="ast"
# heading nodes from a previously-bloated graph are dropped by the
# full-rebuild AST ownership rule while the shrink accounting
# below still counts the doc as a rebuilt source.
# (#1915). They remain in code_files for corpus membership and
# shrink accounting, but because they are not extract targets the
# full-rebuild AST ownership rule (scoped to
# rebuilt_source_identities, #2333 COEXIST) leaves their existing
# AST heading layer intact alongside the semantic layer.
extract_targets = [p for p in code_files if p not in semantic_doc_files]
commit = _git_head(cwd=watch_root)
+136
View File
@@ -1046,6 +1046,142 @@ def test_build_merge_replaces_changed_file_stale_edges(tmp_path):
assert ("K", "A") in edges, "unchanged file's edge must survive"
def _write_two_tier_graph(graph_path):
"""A graph where docs/readme.md carries BOTH tiers (#2333 COEXIST): an
AST layer (document/heading nodes + a contains edge, _origin=ast) and a
semantic layer (an unstamped concept node, source_location=None)."""
data = {
"directed": False,
"nodes": [
{"id": "docs_readme", "label": "Readme", "file_type": "document",
"source_file": "docs/readme.md", "source_location": "L1",
"_origin": "ast"},
{"id": "docs_readme_intro", "label": "Intro", "file_type": "document",
"source_file": "docs/readme.md", "source_location": "L3",
"_origin": "ast"},
{"id": "auth_flow", "label": "Auth Flow", "file_type": "concept",
"source_file": "docs/readme.md", "source_location": None},
],
"links": [
{"source": "docs_readme", "target": "docs_readme_intro",
"relation": "contains", "confidence": "EXTRACTED",
"source_file": "docs/readme.md", "source_location": "L3",
"weight": 1.0, "_origin": "ast"},
],
"hyperedges": [
{"id": "auth_group", "label": "Auth Group",
"nodes": ["docs_readme", "auth_flow"], "relation": "form",
"confidence": "INFERRED", "source_file": "docs/readme.md"},
],
}
graph_path.write_text(json.dumps(data), encoding="utf-8")
def test_build_merge_semantic_reextract_preserves_ast_layer(tmp_path):
"""#2333/#2336 (COEXIST): a semantic-only re-extract of a file replaces
only that file's SEMANTIC tier — its AST document/heading nodes and AST
edges must survive. Before the fix, replace-by-source was tier-blind and
the semantic chunk deleted the doc's AST layer."""
graph_path = tmp_path / "graph.json"
_write_two_tier_graph(graph_path)
# Semantic-only chunk for the same file: no _origin, null source_location
# (the extraction-spec shape — the semantic path never stamps _origin).
chunk = {"nodes": [
{"id": "session_model", "label": "Session Model", "file_type": "concept",
"source_file": "docs/readme.md", "source_location": None},
], "edges": []}
G = build_merge([chunk], graph_path, dedup=False)
assert "docs_readme" in G and "docs_readme_intro" in G, (
"AST layer deleted by a semantic-only re-extract of the same file"
)
assert G.has_edge("docs_readme", "docs_readme_intro"), (
"AST contains edge deleted by a semantic-only re-extract"
)
assert "auth_flow" not in G, "old semantic node must be replaced"
assert "session_model" in G, "new semantic node must be present"
def test_build_merge_ast_reextract_preserves_semantic_layer(tmp_path):
"""#2333/#2336 inverse: an AST-only re-extract of a file replaces only
that file's AST tier — its semantic concept nodes (and semantic-tier
hyperedges, which an AST pass cannot regenerate) must survive."""
graph_path = tmp_path / "graph.json"
_write_two_tier_graph(graph_path)
# AST-only re-extract: the doc changed — Intro heading replaced by
# Quickstart. extract() always stamps _origin=ast.
chunk = {"nodes": [
{"id": "docs_readme", "label": "Readme", "file_type": "document",
"source_file": "docs/readme.md", "source_location": "L1",
"_origin": "ast"},
{"id": "docs_readme_quickstart", "label": "Quickstart",
"file_type": "document", "source_file": "docs/readme.md",
"source_location": "L5", "_origin": "ast"},
], "edges": [
{"source": "docs_readme", "target": "docs_readme_quickstart",
"relation": "contains", "confidence": "EXTRACTED",
"source_file": "docs/readme.md", "source_location": "L5",
"weight": 1.0, "_origin": "ast"},
]}
G = build_merge([chunk], graph_path, dedup=False)
assert "auth_flow" in G, (
"semantic layer deleted by an AST-only re-extract of the same file"
)
assert "docs_readme_intro" not in G, "stale AST node must be replaced"
assert "docs_readme_quickstart" in G, "fresh AST node must be present"
assert G.has_edge("docs_readme", "docs_readme_quickstart")
carried_he_ids = {he.get("id") for he in G.graph.get("hyperedges", [])}
assert "auth_group" in carried_he_ids, (
"semantic-tier hyperedge dropped by an AST-only re-extract (#2336)"
)
def test_merge_raw_extraction_tier_scoped(tmp_path):
"""#2333 raw-path mirror: merge_raw_extraction (extract --no-cluster
incremental) applies the same tier-scoped replace as build_merge a
semantic-only re-extract keeps the file's AST nodes/edges."""
from graphify.build import merge_raw_extraction
graph_path = tmp_path / "graph.json"
_write_two_tier_graph(graph_path)
new = {"nodes": [
{"id": "session_model", "label": "Session Model", "file_type": "concept",
"source_file": "docs/readme.md", "source_location": None},
], "edges": [], "input_tokens": 0, "output_tokens": 0}
out = merge_raw_extraction(new, graph_path)
node_ids = {n["id"] for n in out["nodes"]}
assert {"docs_readme", "docs_readme_intro", "session_model"} <= node_ids, (
"AST layer dropped by a semantic-only raw re-extract"
)
assert "auth_flow" not in node_ids, "old semantic node must be replaced"
edge_keys = {(e.get("source"), e.get("target")) for e in out["edges"]}
assert ("docs_readme", "docs_readme_intro") in edge_keys, (
"AST edge dropped by a semantic-only raw re-extract"
)
he_ids = {he.get("id") for he in out.get("hyperedges", [])}
assert "auth_group" not in he_ids, (
"semantic-tier hyperedge must be replaced by a semantic re-extract"
)
def test_is_ast_tier_legacy_fallback():
"""#2334: _origin wins when present; unstamped legacy items fall back to
the source_location shape (AST emits 'L<line>', semantic emits null)."""
from graphify.build import _is_ast_tier
assert _is_ast_tier({"_origin": "ast"}) is True
assert _is_ast_tier({"_origin": "ast", "source_location": None}) is True
assert _is_ast_tier({"source_location": "L10"}) is True
assert _is_ast_tier({"source_location": None}) is False
assert _is_ast_tier({}) is False
assert _is_ast_tier({"_origin": "semantic", "source_location": "L10"}) is False
def test_build_merge_root_collapses_convention_drift(tmp_path):
"""Skill contract: the extraction subagent must emit source_file as the
verbatim path from FILE_LIST AND the caller must pass root= (the build root).
+98 -20
View File
@@ -2050,11 +2050,13 @@ def test_rebuild_code_quick_scans_doc_without_semantic_nodes(tmp_path):
assert {"notes", "notes_alpha", "notes_beta"} <= ids
def test_rebuild_code_polluted_graph_self_heals_on_full_rebuild(tmp_path):
"""#1915: a graph already bloated by the bug (semantic doc nodes PLUS stale
_origin=="ast" heading nodes for the same doc) sheds the heading nodes on
the next full rebuild via the AST ownership rule and the shrink guard
accepts the smaller write without --force."""
def test_full_rebuild_preserves_semantic_backed_doc_ast_layer(tmp_path):
"""#2333 (COEXIST): a doc that carries BOTH an AST heading layer and a
semantic layer keeps both across a full rebuild. The doc is excluded from
extract_targets (#1915, no re-quick-scan), so its AST nodes are NOT
regenerated this run the full-rebuild ownership rule must therefore not
drop them (it owns only rebuilt sources, not everything in watch_root).
Supersedes the pre-COEXIST #1915 self-heal, which deleted the AST layer."""
from graphify.watch import _rebuild_code
corpus = tmp_path / "corpus"
@@ -2070,31 +2072,107 @@ def test_rebuild_code_polluted_graph_self_heals_on_full_rebuild(tmp_path):
graph_path = corpus / "graphify-out" / "graph.json"
data = json.loads(graph_path.read_text(encoding="utf-8"))
assert _AST_GUIDE_IDS <= {n["id"] for n in data["nodes"]}
doc_nodes_before = sum(
1 for n in data["nodes"] if n.get("source_file") == "guide.md"
)
# Layer the semantic representation on top -> the double-represented state.
data["nodes"].extend([
{"id": "guide_doc", "label": "Guide", "file_type": "document",
"source_file": "guide.md"},
# Layer the semantic representation on top — both tiers now coexist.
data["nodes"].append(
{"id": "auth_flow", "label": "Auth Flow", "file_type": "concept",
"source_file": "guide.md"},
])
data["links"].append({
"source": "guide_doc", "target": "auth_flow", "relation": "explains",
"confidence": "INFERRED", "source_file": "guide.md",
})
)
graph_path.write_text(json.dumps(data), encoding="utf-8")
nodes_before = len(data["nodes"])
# No force=True: the self-heal shrink must be accepted by the guard.
# Full rebuild WITHOUT force: guide.md is now semantic-backed, so it is
# excluded from extract_targets — its existing AST layer must survive.
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
after = json.loads(graph_path.read_text(encoding="utf-8"))
after_ids = {n["id"] for n in after["nodes"]}
assert {"guide_doc", "auth_flow"} <= after_ids
assert not (_AST_GUIDE_IDS & after_ids), (
"stale AST heading nodes for a semantic-backed doc must self-heal away"
assert "auth_flow" in after_ids, "semantic layer lost on full rebuild"
assert _AST_GUIDE_IDS <= after_ids, (
"AST heading layer of a semantic-backed doc dropped by a full "
"rebuild (#2333 COEXIST)"
)
doc_nodes_after = sum(
1 for n in after["nodes"]
if n.get("source_file") == "guide.md" and n["id"] != "auth_flow"
)
assert doc_nodes_after == doc_nodes_before, (
"document-node count changed across a full rebuild of a "
f"semantic-backed doc: {doc_nodes_before} -> {doc_nodes_after}"
)
def test_full_rebuild_regenerates_docs_with_legacy_unstamped_nodes(tmp_path):
"""#2334: a legacy heading node without the _origin stamp (pre-0.9.16
graph) must not fake a semantic layer _is_ast_tier's shape fallback
(source_location "L<n>") classifies it as AST, so the doc stays in
extract_targets, is re-quick-scanned, and comes back fully re-stamped."""
from graphify.watch import _rebuild_code
corpus = tmp_path / "corpus"
corpus.mkdir()
(corpus / "app.py").write_text(
"def handle_login():\n return 1\n", encoding="utf-8"
)
(corpus / "guide.md").write_text(
"# Overview\n\n## Setup\n\n## Usage\n", encoding="utf-8"
)
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
graph_path = corpus / "graphify-out" / "graph.json"
data = json.loads(graph_path.read_text(encoding="utf-8"))
assert _AST_GUIDE_IDS <= {n["id"] for n in data["nodes"]}
# Simulate a legacy graph: strip the _origin stamp from one heading node.
stripped = next(
n for n in data["nodes"] if n["id"] == "guide_overview"
)
stripped.pop("_origin", None)
graph_path.write_text(json.dumps(data), encoding="utf-8")
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
after = json.loads(graph_path.read_text(encoding="utf-8"))
after_by_id = {n["id"]: n for n in after["nodes"]}
assert _AST_GUIDE_IDS <= set(after_by_id), (
"legacy unstamped heading node made the doc look semantic-backed "
"and its AST structure was dropped (#2334)"
)
for nid in _AST_GUIDE_IDS:
assert after_by_id[nid].get("_origin") == "ast", (
f"{nid} not re-stamped with _origin=ast after the full rebuild"
)
def test_full_rebuild_drops_stale_ast_for_reextracted_code(tmp_path):
"""#1116 guard: tier-scoping the full-rebuild ownership rule (#2333) must
not stop a genuinely re-extracted code file from shedding its stale AST
nodes a renamed function's old symbol node still disappears."""
from graphify.watch import _rebuild_code
corpus = tmp_path / "corpus"
corpus.mkdir()
(corpus / "app.py").write_text(
"def old_name():\n return 1\n", encoding="utf-8"
)
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
graph_path = corpus / "graphify-out" / "graph.json"
ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]}
assert "app_old_name" in ids
(corpus / "app.py").write_text(
"def new_name():\n return 1\n", encoding="utf-8"
)
# Full rebuild (no changed_paths): app.py is re-extracted, so its stale
# AST symbol node is owned by this run and must be dropped.
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]}
assert "app_new_name" in ids
assert "app_old_name" not in ids, (
"stale AST node for a re-extracted code file survived (#1116)"
)
assert len(after["nodes"]) < nodes_before, "polluted graph should shrink"
# ── #2014: code-typed semantic nodes count as a doc's semantic layer ───────────