mirror of
https://github.com/safishamsi/graphify.git
synced 2026-08-28 17:26:48 +00:00
Node/edge ownership was keyed on source_file alone, but each file has an AST tier and a semantic tier. A semantic-only re-extract deleted a doc's AST headings, and a full _rebuild_code deleted document AST nodes. Add an _is_ast_tier() predicate (with a source_location legacy fallback for the unreliable _origin marker, #2334), backfill _origin on load so graphs self-heal, make build_merge / merge_raw_extraction replace by (source_file, tier) instead of source_file (coexist policy: an AST re-extract replaces only AST nodes and keeps the semantic layer, and vice versa), and scope the full-rebuild AST drop to sources actually regenerated (#2336) so a semantic-backed doc keeps its AST layer. Deletion prune stays tier-blind. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
c262478792
commit
97656fae2d
+89
-21
@@ -34,6 +34,23 @@ from .paths import default_graph_json as _default_graph_json
|
||||
from .validate import validate_extraction
|
||||
|
||||
|
||||
# Deterministic (AST) extractors emit source_location "L<line>"; the semantic
|
||||
# extraction spec emits null. Used by _is_ast_tier as a shape fallback for
|
||||
# legacy items that predate the _origin marker (#2334).
|
||||
_AST_LOC_RE = re.compile(r"^L\d")
|
||||
|
||||
|
||||
def _is_ast_tier(item: dict) -> bool:
|
||||
"""AST vs semantic tier. _origin wins when present; unstamped legacy items
|
||||
(pre-0.9.16) fall back to shape: deterministic extractors emit
|
||||
source_location 'L<line>', the semantic spec emits null (#2334)."""
|
||||
o = item.get("_origin")
|
||||
if o is not None:
|
||||
return o == "ast"
|
||||
loc = item.get("source_location")
|
||||
return isinstance(loc, str) and bool(_AST_LOC_RE.match(loc))
|
||||
|
||||
|
||||
# Language interop families, keyed by extension, for the cross-language phantom-edge
|
||||
# guard in the edge loop below. Families group by REAL interop (JS/TS share a module
|
||||
# graph; C/C++/ObjC share a compilation unit via headers; JVM langs share bytecode),
|
||||
@@ -454,7 +471,7 @@ def _semantic_id_remap(nodes: list, root: str | None) -> dict:
|
||||
for node in nodes:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
if node.get("_origin") == "ast":
|
||||
if _is_ast_tier(node):
|
||||
continue
|
||||
nid = node.get("id")
|
||||
sf = node.get("source_file")
|
||||
@@ -781,6 +798,13 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat
|
||||
sf = str(attrs.get("source_file", ""))
|
||||
if not label or not sf:
|
||||
continue
|
||||
# Strict _origin check on purpose — NOT _is_ast_tier (#2334): existing
|
||||
# graph items are backfilled with _origin at load time, so inside a
|
||||
# build the only unstamped items are fresh SEMANTIC chunks (extract()
|
||||
# always stamps AST output). Those may carry drifted 'L<line>'
|
||||
# source_locations (the very ghosts #1145-extended collapses), and the
|
||||
# shape fallback would misread them as AST — turning two same-file LLM
|
||||
# duplicates into a fake AST/AST collision that blocks their merge.
|
||||
is_ast = attrs.get("_origin") == "ast"
|
||||
if attrs.get("source_location") or is_ast:
|
||||
# Key on the FULL normalized source_file, not the bare basename
|
||||
@@ -809,7 +833,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat
|
||||
for nid in sorted(node_set):
|
||||
attrs = G.nodes[nid]
|
||||
if attrs.get("_origin") == "ast":
|
||||
continue # AST nodes are never ghosts
|
||||
continue # AST nodes are never ghosts (strict check — see Pass 1)
|
||||
label = str(attrs.get("label", "")).strip()
|
||||
sf = str(attrs.get("source_file", ""))
|
||||
if not label or not sf:
|
||||
@@ -1216,9 +1240,23 @@ def _load_existing_graph(graph_path: Path) -> "tuple[list, list, list, bool] | N
|
||||
"Delete the file and run a full rebuild."
|
||||
) from exc
|
||||
links_key = "links" if "links" in data else "edges"
|
||||
nodes = list(data.get("nodes", []))
|
||||
edges = list(data.get(links_key, []))
|
||||
# Backfill tier provenance on legacy items (#2334): _origin is stamped at
|
||||
# extraction time only (extract.py for AST, and the semantic path never
|
||||
# stamps), so pre-0.9.16 graphs and externally-merged fragments carry
|
||||
# unstamped items. Stamp them via the _is_ast_tier shape fallback so the
|
||||
# graph self-heals on the next write and every downstream tier decision
|
||||
# (build_merge replace, watch reconcile) reads an explicit marker.
|
||||
for item in nodes:
|
||||
if isinstance(item, dict):
|
||||
item.setdefault("_origin", "ast" if _is_ast_tier(item) else "semantic")
|
||||
for item in edges:
|
||||
if isinstance(item, dict):
|
||||
item.setdefault("_origin", "ast" if _is_ast_tier(item) else "semantic")
|
||||
return (
|
||||
list(data.get("nodes", [])),
|
||||
list(data.get(links_key, [])),
|
||||
nodes,
|
||||
edges,
|
||||
list(data.get("hyperedges", [])),
|
||||
bool(data.get("directed", False)),
|
||||
)
|
||||
@@ -1236,8 +1274,10 @@ def merge_raw_extraction(
|
||||
Replace/prune semantics mirror :func:`build_merge` exactly, so the raw and
|
||||
clustered incremental paths can't drift:
|
||||
|
||||
- sources re-extracted this run REPLACE their prior contribution — existing
|
||||
nodes/edges/hyperedges owned by them are dropped, matched in both raw and
|
||||
- sources re-extracted this run REPLACE their prior contribution PER TIER
|
||||
(#2333/#2336): existing nodes/edges/hyperedges owned by them are dropped
|
||||
only when the new extraction contains the same tier (AST vs semantic,
|
||||
per :func:`_is_ast_tier`) for that source, matched in both raw and
|
||||
:func:`_norm_source_file` form (#1007);
|
||||
- ``prune_sources`` (deleted / excluded / graph-stale files) are dropped,
|
||||
with the ``_abs_identity`` third-form fallback (#2012), and "replace" wins
|
||||
@@ -1265,17 +1305,24 @@ def merge_raw_extraction(
|
||||
else _infer_merge_root(graph_path)
|
||||
)
|
||||
|
||||
new_sources: set[str] = set()
|
||||
# Tier-scoped replace, mirroring build_merge (#2333/#2336, COEXIST): a
|
||||
# source re-extracted this run replaces only the tier(s) actually present
|
||||
# in the new extraction, so an AST-only re-extract keeps the file's
|
||||
# semantic layer and vice versa.
|
||||
new_ast_sources: set[str] = set()
|
||||
new_sem_sources: set[str] = set()
|
||||
for n in new.get("nodes", []):
|
||||
if not isinstance(n, dict):
|
||||
continue
|
||||
sf = n.get("source_file")
|
||||
if not sf:
|
||||
continue
|
||||
new_sources.add(sf)
|
||||
tier_sources = new_ast_sources if _is_ast_tier(n) else new_sem_sources
|
||||
tier_sources.add(sf)
|
||||
norm = _norm_source_file(sf, _eff_root)
|
||||
if norm:
|
||||
new_sources.add(norm)
|
||||
tier_sources.add(norm)
|
||||
new_sources: set[str] = new_ast_sources | new_sem_sources
|
||||
|
||||
prune_set: set[str] = set()
|
||||
prune_abs: set[str] = set()
|
||||
@@ -1300,7 +1347,12 @@ def merge_raw_extraction(
|
||||
if not isinstance(item, dict):
|
||||
return True
|
||||
sf = item.get("source_file")
|
||||
if sf in new_sources or _norm_source_file(sf, _eff_root) in new_sources:
|
||||
# Tier-scoped replace: an item is superseded only when ITS OWN tier
|
||||
# re-extracted its source. Hyperedges are semantic-tier (no _origin,
|
||||
# null source_location), so an AST-only re-extract carries them.
|
||||
# Deletion pruning below stays tier-blind.
|
||||
own = new_ast_sources if _is_ast_tier(item) else new_sem_sources
|
||||
if sf in own or _norm_source_file(sf, _eff_root) in own:
|
||||
return True # re-extracted this run — replaced by the new chunk
|
||||
if not sf:
|
||||
return False # unowned — carry forward
|
||||
@@ -1332,11 +1384,13 @@ def build_merge(
|
||||
) -> nx.Graph:
|
||||
"""Load existing graph.json, merge new chunks into it, and save back.
|
||||
|
||||
Re-extracted files REPLACE their prior contribution: any source_file present
|
||||
in new_chunks is dropped from the loaded graph before merging, so a changed
|
||||
file's stale nodes/edges don't accumulate. Files absent from new_chunks are
|
||||
preserved unchanged; deleted files are removed via prune_sources.
|
||||
Safe to call repeatedly.
|
||||
Re-extracted files REPLACE their prior contribution per tier (#2333/#2336):
|
||||
a source_file present in new_chunks has its existing nodes/edges dropped
|
||||
for each tier (AST vs semantic, per :func:`_is_ast_tier`) the new chunks
|
||||
actually contain, so a changed file's stale nodes/edges don't accumulate
|
||||
while a one-tier re-extract keeps the other tier's layer intact. Files
|
||||
absent from new_chunks are preserved unchanged; deleted files are removed
|
||||
via prune_sources (tier-blind). Safe to call repeatedly.
|
||||
root: if given, absolute source_file paths in new_chunks are made relative (#932).
|
||||
directed: if None (default), honor the on-disk graph's own ``directed`` flag
|
||||
when one exists, so an incremental merge can't silently flip a directed
|
||||
@@ -1379,21 +1433,32 @@ def build_merge(
|
||||
# for them; genuinely deleted files are still handled via prune_sources.
|
||||
# Matched in both raw and _norm_source_file form because new_chunks may carry
|
||||
# absolute win32 paths while the stored graph keeps relative posix (#1007).
|
||||
# Replacement is tier-scoped (#2333/#2336, COEXIST): each file has two
|
||||
# producers — the deterministic AST pass and the semantic/LLM pass — whose
|
||||
# node sets coexist in the graph. A re-extract of one tier must replace
|
||||
# only that tier's prior contribution, never the other's (a semantic-only
|
||||
# chunk used to delete the file's AST headings). Which tier a NEW chunk
|
||||
# item belongs to is read via _is_ast_tier (existing items were stamped by
|
||||
# _load_existing_graph above).
|
||||
_replace_root = _eff_root
|
||||
new_sources: set[str] = set()
|
||||
new_ast_sources: set[str] = set()
|
||||
new_sem_sources: set[str] = set()
|
||||
for ch in new_chunks:
|
||||
for n in ch.get("nodes", []):
|
||||
sf = n.get("source_file")
|
||||
if not sf:
|
||||
continue
|
||||
new_sources.add(sf)
|
||||
tier_sources = new_ast_sources if _is_ast_tier(n) else new_sem_sources
|
||||
tier_sources.add(sf)
|
||||
norm = _norm_source_file(sf, _replace_root)
|
||||
if norm:
|
||||
new_sources.add(norm)
|
||||
tier_sources.add(norm)
|
||||
new_sources: set[str] = new_ast_sources | new_sem_sources
|
||||
if new_sources:
|
||||
def _kept(item: dict) -> bool:
|
||||
sf = item.get("source_file")
|
||||
return sf not in new_sources and _norm_source_file(sf, _replace_root) not in new_sources
|
||||
own = new_ast_sources if _is_ast_tier(item) else new_sem_sources
|
||||
return sf not in own and _norm_source_file(sf, _replace_root) not in own
|
||||
existing_nodes = [n for n in existing_nodes if _kept(n)]
|
||||
existing_edges = [e for e in existing_edges if _kept(e)]
|
||||
|
||||
@@ -1461,8 +1526,11 @@ def build_merge(
|
||||
continue
|
||||
sf = he.get("source_file")
|
||||
norm = _norm_source_file(sf, _eff_root)
|
||||
if sf in new_sources or norm in new_sources:
|
||||
continue # re-extracted — replaced by the new chunk's version
|
||||
# Hyperedges are semantic-tier: only a SEMANTIC re-extract of the
|
||||
# source replaces them. An AST-only re-extract cannot regenerate
|
||||
# hyperedges, so dropping them there would be data loss (#2336).
|
||||
if sf in new_sem_sources or norm in new_sem_sources:
|
||||
continue # semantically re-extracted — replaced by the new chunk's version
|
||||
if _prune_match(sf):
|
||||
continue # deleted — pruned
|
||||
carried.append(he)
|
||||
|
||||
+38
-16
@@ -491,6 +491,16 @@ def _reconcile_existing_graph(
|
||||
existing = json.loads(existing_graph.read_text(encoding="utf-8"))
|
||||
existing_graph_data = existing
|
||||
|
||||
# Backfill tier provenance on legacy items (#2334), mirroring
|
||||
# build._load_existing_graph (this reconcile path loads the raw dict
|
||||
# separately, so the backfill there does not reach it). Stamping preserved
|
||||
# items means the graph self-heals on this write.
|
||||
from graphify.build import _is_ast_tier
|
||||
for _bucket in ("nodes", "links", "edges"):
|
||||
for _item in existing.get(_bucket, []):
|
||||
if isinstance(_item, dict):
|
||||
_item.setdefault("_origin", "ast" if _is_ast_tier(_item) else "semantic")
|
||||
|
||||
try:
|
||||
from graphify.build import _norm_source_file as _nsf
|
||||
from graphify.extract import _get_extractor
|
||||
@@ -584,15 +594,20 @@ def _reconcile_existing_graph(
|
||||
"Run a full re-extraction to purge them if the exclusion is intentional."
|
||||
)
|
||||
|
||||
# A full re-extraction owns every AST node under watch_root. Incremental
|
||||
# extraction owns only nodes from rebuilt or deleted sources. Semantic
|
||||
# nodes lack the AST origin marker and remain preserved.
|
||||
# A full re-extraction owns the AST nodes of every source it actually
|
||||
# re-extracted (extract_targets, via rebuilt_source_identities) — NOT
|
||||
# every AST node under watch_root: a semantic-backed doc is excluded
|
||||
# from extract_targets (#1915), so its existing AST layer is not
|
||||
# regenerated this run and dropping it would be data loss (#2333,
|
||||
# COEXIST — the AST and semantic layers of a file coexist).
|
||||
# Incremental extraction owns only nodes from rebuilt or deleted
|
||||
# sources. Semantic-tier nodes (per _is_ast_tier) remain preserved.
|
||||
preserved_nodes = [
|
||||
node
|
||||
for node in existing.get("nodes", [])
|
||||
if node["id"] not in new_ast_ids
|
||||
and not (
|
||||
node.get("_origin") == "ast"
|
||||
_is_ast_tier(node)
|
||||
and (
|
||||
(
|
||||
not node.get("source_file")
|
||||
@@ -600,7 +615,7 @@ def _reconcile_existing_graph(
|
||||
)
|
||||
or (
|
||||
full_rebuild
|
||||
and source_paths.in_watch_root(node.get("source_file"))
|
||||
and source_paths.is_evicted(node, rebuilt_source_identities)
|
||||
)
|
||||
)
|
||||
)
|
||||
@@ -621,7 +636,7 @@ def _reconcile_existing_graph(
|
||||
and edge.get("target") in all_ids
|
||||
and not source_paths.is_evicted(edge, edge_evicted_source_identities)
|
||||
and not (
|
||||
edge.get("_origin") == "ast"
|
||||
_is_ast_tier(edge)
|
||||
and source_paths.is_evicted(edge, rebuilt_source_identities)
|
||||
)
|
||||
]
|
||||
@@ -973,7 +988,7 @@ def _rebuild_code(
|
||||
try:
|
||||
from graphify.extract import extract, _get_extractor
|
||||
from graphify.detect import detect
|
||||
from graphify.build import build_from_json, _norm_source_file as _nsf
|
||||
from graphify.build import build_from_json, _is_ast_tier, _norm_source_file as _nsf
|
||||
from graphify.cluster import cluster, remap_communities_to_previous, score_all
|
||||
from graphify.analyze import god_nodes, surprising_connections, suggest_questions
|
||||
from graphify.report import generate
|
||||
@@ -1008,10 +1023,12 @@ def _rebuild_code(
|
||||
# existing graph must not ALSO be AST-quick-scanned — otherwise every
|
||||
# rebuild mints heading nodes on top of the preserved semantic nodes
|
||||
# and the doc is represented twice (~4x graph bloat vs the CLI update
|
||||
# path, which AST-extracts only code). Semantic supersedes AST per doc
|
||||
# source: the quick-scan stays as a fallback for docs with no semantic
|
||||
# layer (the no-LLM doc-structure feature, #09b33b7) and for brand-new
|
||||
# docs the graph has never seen. These docs stay in ``code_files`` so
|
||||
# path, which AST-extracts only code). A semantic-backed doc is never
|
||||
# re-quick-scanned, and any AST layer it already carries coexists and
|
||||
# is preserved rather than regenerated (#2333, COEXIST); the
|
||||
# quick-scan stays as a fallback for docs with no semantic layer (the
|
||||
# no-LLM doc-structure feature, #09b33b7) and for brand-new docs the
|
||||
# graph has never seen. These docs stay in ``code_files`` so
|
||||
# corpus membership (#1795 fail-closed deletion evidence) and the
|
||||
# shrink accounting below still cover them — a previously-bloated
|
||||
# graph must be allowed to self-heal on a full rebuild without the
|
||||
@@ -1044,7 +1061,11 @@ def _rebuild_code(
|
||||
# "document" nodes (extractors/markdown.py). "image" stays out.
|
||||
semantic_doc_identities: set[str] = set()
|
||||
for node in prior.get("nodes", []):
|
||||
if node.get("_origin") == "ast":
|
||||
# _is_ast_tier, not a raw _origin check (#2334): a legacy
|
||||
# unstamped AST heading node (source_location "L<n>") must
|
||||
# not fake a semantic layer, or the doc would be excluded
|
||||
# from the AST quick-scan forever.
|
||||
if _is_ast_tier(node):
|
||||
continue
|
||||
if node.get("file_type") not in (
|
||||
"document", "concept", "rationale", "paper", "code"
|
||||
@@ -1129,10 +1150,11 @@ def _rebuild_code(
|
||||
extract_targets = wanted
|
||||
else:
|
||||
# Full rebuild: skip the AST quick-scan for semantic-backed docs
|
||||
# (#1915). They remain in code_files, so stale _origin=="ast"
|
||||
# heading nodes from a previously-bloated graph are dropped by the
|
||||
# full-rebuild AST ownership rule while the shrink accounting
|
||||
# below still counts the doc as a rebuilt source.
|
||||
# (#1915). They remain in code_files for corpus membership and
|
||||
# shrink accounting, but because they are not extract targets the
|
||||
# full-rebuild AST ownership rule (scoped to
|
||||
# rebuilt_source_identities, #2333 COEXIST) leaves their existing
|
||||
# AST heading layer intact alongside the semantic layer.
|
||||
extract_targets = [p for p in code_files if p not in semantic_doc_files]
|
||||
|
||||
commit = _git_head(cwd=watch_root)
|
||||
|
||||
@@ -1046,6 +1046,142 @@ def test_build_merge_replaces_changed_file_stale_edges(tmp_path):
|
||||
assert ("K", "A") in edges, "unchanged file's edge must survive"
|
||||
|
||||
|
||||
def _write_two_tier_graph(graph_path):
|
||||
"""A graph where docs/readme.md carries BOTH tiers (#2333 COEXIST): an
|
||||
AST layer (document/heading nodes + a contains edge, _origin=ast) and a
|
||||
semantic layer (an unstamped concept node, source_location=None)."""
|
||||
data = {
|
||||
"directed": False,
|
||||
"nodes": [
|
||||
{"id": "docs_readme", "label": "Readme", "file_type": "document",
|
||||
"source_file": "docs/readme.md", "source_location": "L1",
|
||||
"_origin": "ast"},
|
||||
{"id": "docs_readme_intro", "label": "Intro", "file_type": "document",
|
||||
"source_file": "docs/readme.md", "source_location": "L3",
|
||||
"_origin": "ast"},
|
||||
{"id": "auth_flow", "label": "Auth Flow", "file_type": "concept",
|
||||
"source_file": "docs/readme.md", "source_location": None},
|
||||
],
|
||||
"links": [
|
||||
{"source": "docs_readme", "target": "docs_readme_intro",
|
||||
"relation": "contains", "confidence": "EXTRACTED",
|
||||
"source_file": "docs/readme.md", "source_location": "L3",
|
||||
"weight": 1.0, "_origin": "ast"},
|
||||
],
|
||||
"hyperedges": [
|
||||
{"id": "auth_group", "label": "Auth Group",
|
||||
"nodes": ["docs_readme", "auth_flow"], "relation": "form",
|
||||
"confidence": "INFERRED", "source_file": "docs/readme.md"},
|
||||
],
|
||||
}
|
||||
graph_path.write_text(json.dumps(data), encoding="utf-8")
|
||||
|
||||
|
||||
def test_build_merge_semantic_reextract_preserves_ast_layer(tmp_path):
|
||||
"""#2333/#2336 (COEXIST): a semantic-only re-extract of a file replaces
|
||||
only that file's SEMANTIC tier — its AST document/heading nodes and AST
|
||||
edges must survive. Before the fix, replace-by-source was tier-blind and
|
||||
the semantic chunk deleted the doc's AST layer."""
|
||||
graph_path = tmp_path / "graph.json"
|
||||
_write_two_tier_graph(graph_path)
|
||||
|
||||
# Semantic-only chunk for the same file: no _origin, null source_location
|
||||
# (the extraction-spec shape — the semantic path never stamps _origin).
|
||||
chunk = {"nodes": [
|
||||
{"id": "session_model", "label": "Session Model", "file_type": "concept",
|
||||
"source_file": "docs/readme.md", "source_location": None},
|
||||
], "edges": []}
|
||||
G = build_merge([chunk], graph_path, dedup=False)
|
||||
|
||||
assert "docs_readme" in G and "docs_readme_intro" in G, (
|
||||
"AST layer deleted by a semantic-only re-extract of the same file"
|
||||
)
|
||||
assert G.has_edge("docs_readme", "docs_readme_intro"), (
|
||||
"AST contains edge deleted by a semantic-only re-extract"
|
||||
)
|
||||
assert "auth_flow" not in G, "old semantic node must be replaced"
|
||||
assert "session_model" in G, "new semantic node must be present"
|
||||
|
||||
|
||||
def test_build_merge_ast_reextract_preserves_semantic_layer(tmp_path):
|
||||
"""#2333/#2336 inverse: an AST-only re-extract of a file replaces only
|
||||
that file's AST tier — its semantic concept nodes (and semantic-tier
|
||||
hyperedges, which an AST pass cannot regenerate) must survive."""
|
||||
graph_path = tmp_path / "graph.json"
|
||||
_write_two_tier_graph(graph_path)
|
||||
|
||||
# AST-only re-extract: the doc changed — Intro heading replaced by
|
||||
# Quickstart. extract() always stamps _origin=ast.
|
||||
chunk = {"nodes": [
|
||||
{"id": "docs_readme", "label": "Readme", "file_type": "document",
|
||||
"source_file": "docs/readme.md", "source_location": "L1",
|
||||
"_origin": "ast"},
|
||||
{"id": "docs_readme_quickstart", "label": "Quickstart",
|
||||
"file_type": "document", "source_file": "docs/readme.md",
|
||||
"source_location": "L5", "_origin": "ast"},
|
||||
], "edges": [
|
||||
{"source": "docs_readme", "target": "docs_readme_quickstart",
|
||||
"relation": "contains", "confidence": "EXTRACTED",
|
||||
"source_file": "docs/readme.md", "source_location": "L5",
|
||||
"weight": 1.0, "_origin": "ast"},
|
||||
]}
|
||||
G = build_merge([chunk], graph_path, dedup=False)
|
||||
|
||||
assert "auth_flow" in G, (
|
||||
"semantic layer deleted by an AST-only re-extract of the same file"
|
||||
)
|
||||
assert "docs_readme_intro" not in G, "stale AST node must be replaced"
|
||||
assert "docs_readme_quickstart" in G, "fresh AST node must be present"
|
||||
assert G.has_edge("docs_readme", "docs_readme_quickstart")
|
||||
carried_he_ids = {he.get("id") for he in G.graph.get("hyperedges", [])}
|
||||
assert "auth_group" in carried_he_ids, (
|
||||
"semantic-tier hyperedge dropped by an AST-only re-extract (#2336)"
|
||||
)
|
||||
|
||||
|
||||
def test_merge_raw_extraction_tier_scoped(tmp_path):
|
||||
"""#2333 raw-path mirror: merge_raw_extraction (extract --no-cluster
|
||||
incremental) applies the same tier-scoped replace as build_merge — a
|
||||
semantic-only re-extract keeps the file's AST nodes/edges."""
|
||||
from graphify.build import merge_raw_extraction
|
||||
|
||||
graph_path = tmp_path / "graph.json"
|
||||
_write_two_tier_graph(graph_path)
|
||||
|
||||
new = {"nodes": [
|
||||
{"id": "session_model", "label": "Session Model", "file_type": "concept",
|
||||
"source_file": "docs/readme.md", "source_location": None},
|
||||
], "edges": [], "input_tokens": 0, "output_tokens": 0}
|
||||
out = merge_raw_extraction(new, graph_path)
|
||||
|
||||
node_ids = {n["id"] for n in out["nodes"]}
|
||||
assert {"docs_readme", "docs_readme_intro", "session_model"} <= node_ids, (
|
||||
"AST layer dropped by a semantic-only raw re-extract"
|
||||
)
|
||||
assert "auth_flow" not in node_ids, "old semantic node must be replaced"
|
||||
edge_keys = {(e.get("source"), e.get("target")) for e in out["edges"]}
|
||||
assert ("docs_readme", "docs_readme_intro") in edge_keys, (
|
||||
"AST edge dropped by a semantic-only raw re-extract"
|
||||
)
|
||||
he_ids = {he.get("id") for he in out.get("hyperedges", [])}
|
||||
assert "auth_group" not in he_ids, (
|
||||
"semantic-tier hyperedge must be replaced by a semantic re-extract"
|
||||
)
|
||||
|
||||
|
||||
def test_is_ast_tier_legacy_fallback():
|
||||
"""#2334: _origin wins when present; unstamped legacy items fall back to
|
||||
the source_location shape (AST emits 'L<line>', semantic emits null)."""
|
||||
from graphify.build import _is_ast_tier
|
||||
|
||||
assert _is_ast_tier({"_origin": "ast"}) is True
|
||||
assert _is_ast_tier({"_origin": "ast", "source_location": None}) is True
|
||||
assert _is_ast_tier({"source_location": "L10"}) is True
|
||||
assert _is_ast_tier({"source_location": None}) is False
|
||||
assert _is_ast_tier({}) is False
|
||||
assert _is_ast_tier({"_origin": "semantic", "source_location": "L10"}) is False
|
||||
|
||||
|
||||
def test_build_merge_root_collapses_convention_drift(tmp_path):
|
||||
"""Skill contract: the extraction subagent must emit source_file as the
|
||||
verbatim path from FILE_LIST AND the caller must pass root= (the build root).
|
||||
|
||||
+98
-20
@@ -2050,11 +2050,13 @@ def test_rebuild_code_quick_scans_doc_without_semantic_nodes(tmp_path):
|
||||
assert {"notes", "notes_alpha", "notes_beta"} <= ids
|
||||
|
||||
|
||||
def test_rebuild_code_polluted_graph_self_heals_on_full_rebuild(tmp_path):
|
||||
"""#1915: a graph already bloated by the bug (semantic doc nodes PLUS stale
|
||||
_origin=="ast" heading nodes for the same doc) sheds the heading nodes on
|
||||
the next full rebuild via the AST ownership rule — and the shrink guard
|
||||
accepts the smaller write without --force."""
|
||||
def test_full_rebuild_preserves_semantic_backed_doc_ast_layer(tmp_path):
|
||||
"""#2333 (COEXIST): a doc that carries BOTH an AST heading layer and a
|
||||
semantic layer keeps both across a full rebuild. The doc is excluded from
|
||||
extract_targets (#1915, no re-quick-scan), so its AST nodes are NOT
|
||||
regenerated this run — the full-rebuild ownership rule must therefore not
|
||||
drop them (it owns only rebuilt sources, not everything in watch_root).
|
||||
Supersedes the pre-COEXIST #1915 self-heal, which deleted the AST layer."""
|
||||
from graphify.watch import _rebuild_code
|
||||
|
||||
corpus = tmp_path / "corpus"
|
||||
@@ -2070,31 +2072,107 @@ def test_rebuild_code_polluted_graph_self_heals_on_full_rebuild(tmp_path):
|
||||
graph_path = corpus / "graphify-out" / "graph.json"
|
||||
data = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
assert _AST_GUIDE_IDS <= {n["id"] for n in data["nodes"]}
|
||||
doc_nodes_before = sum(
|
||||
1 for n in data["nodes"] if n.get("source_file") == "guide.md"
|
||||
)
|
||||
|
||||
# Layer the semantic representation on top -> the double-represented state.
|
||||
data["nodes"].extend([
|
||||
{"id": "guide_doc", "label": "Guide", "file_type": "document",
|
||||
"source_file": "guide.md"},
|
||||
# Layer the semantic representation on top — both tiers now coexist.
|
||||
data["nodes"].append(
|
||||
{"id": "auth_flow", "label": "Auth Flow", "file_type": "concept",
|
||||
"source_file": "guide.md"},
|
||||
])
|
||||
data["links"].append({
|
||||
"source": "guide_doc", "target": "auth_flow", "relation": "explains",
|
||||
"confidence": "INFERRED", "source_file": "guide.md",
|
||||
})
|
||||
)
|
||||
graph_path.write_text(json.dumps(data), encoding="utf-8")
|
||||
nodes_before = len(data["nodes"])
|
||||
|
||||
# No force=True: the self-heal shrink must be accepted by the guard.
|
||||
# Full rebuild WITHOUT force: guide.md is now semantic-backed, so it is
|
||||
# excluded from extract_targets — its existing AST layer must survive.
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
|
||||
after = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
after_ids = {n["id"] for n in after["nodes"]}
|
||||
assert {"guide_doc", "auth_flow"} <= after_ids
|
||||
assert not (_AST_GUIDE_IDS & after_ids), (
|
||||
"stale AST heading nodes for a semantic-backed doc must self-heal away"
|
||||
assert "auth_flow" in after_ids, "semantic layer lost on full rebuild"
|
||||
assert _AST_GUIDE_IDS <= after_ids, (
|
||||
"AST heading layer of a semantic-backed doc dropped by a full "
|
||||
"rebuild (#2333 COEXIST)"
|
||||
)
|
||||
doc_nodes_after = sum(
|
||||
1 for n in after["nodes"]
|
||||
if n.get("source_file") == "guide.md" and n["id"] != "auth_flow"
|
||||
)
|
||||
assert doc_nodes_after == doc_nodes_before, (
|
||||
"document-node count changed across a full rebuild of a "
|
||||
f"semantic-backed doc: {doc_nodes_before} -> {doc_nodes_after}"
|
||||
)
|
||||
|
||||
|
||||
def test_full_rebuild_regenerates_docs_with_legacy_unstamped_nodes(tmp_path):
|
||||
"""#2334: a legacy heading node without the _origin stamp (pre-0.9.16
|
||||
graph) must not fake a semantic layer — _is_ast_tier's shape fallback
|
||||
(source_location "L<n>") classifies it as AST, so the doc stays in
|
||||
extract_targets, is re-quick-scanned, and comes back fully re-stamped."""
|
||||
from graphify.watch import _rebuild_code
|
||||
|
||||
corpus = tmp_path / "corpus"
|
||||
corpus.mkdir()
|
||||
(corpus / "app.py").write_text(
|
||||
"def handle_login():\n return 1\n", encoding="utf-8"
|
||||
)
|
||||
(corpus / "guide.md").write_text(
|
||||
"# Overview\n\n## Setup\n\n## Usage\n", encoding="utf-8"
|
||||
)
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
graph_path = corpus / "graphify-out" / "graph.json"
|
||||
data = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
assert _AST_GUIDE_IDS <= {n["id"] for n in data["nodes"]}
|
||||
|
||||
# Simulate a legacy graph: strip the _origin stamp from one heading node.
|
||||
stripped = next(
|
||||
n for n in data["nodes"] if n["id"] == "guide_overview"
|
||||
)
|
||||
stripped.pop("_origin", None)
|
||||
graph_path.write_text(json.dumps(data), encoding="utf-8")
|
||||
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
|
||||
after = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
after_by_id = {n["id"]: n for n in after["nodes"]}
|
||||
assert _AST_GUIDE_IDS <= set(after_by_id), (
|
||||
"legacy unstamped heading node made the doc look semantic-backed "
|
||||
"and its AST structure was dropped (#2334)"
|
||||
)
|
||||
for nid in _AST_GUIDE_IDS:
|
||||
assert after_by_id[nid].get("_origin") == "ast", (
|
||||
f"{nid} not re-stamped with _origin=ast after the full rebuild"
|
||||
)
|
||||
|
||||
|
||||
def test_full_rebuild_drops_stale_ast_for_reextracted_code(tmp_path):
|
||||
"""#1116 guard: tier-scoping the full-rebuild ownership rule (#2333) must
|
||||
not stop a genuinely re-extracted code file from shedding its stale AST
|
||||
nodes — a renamed function's old symbol node still disappears."""
|
||||
from graphify.watch import _rebuild_code
|
||||
|
||||
corpus = tmp_path / "corpus"
|
||||
corpus.mkdir()
|
||||
(corpus / "app.py").write_text(
|
||||
"def old_name():\n return 1\n", encoding="utf-8"
|
||||
)
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
graph_path = corpus / "graphify-out" / "graph.json"
|
||||
ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]}
|
||||
assert "app_old_name" in ids
|
||||
|
||||
(corpus / "app.py").write_text(
|
||||
"def new_name():\n return 1\n", encoding="utf-8"
|
||||
)
|
||||
# Full rebuild (no changed_paths): app.py is re-extracted, so its stale
|
||||
# AST symbol node is owned by this run and must be dropped.
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
|
||||
ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]}
|
||||
assert "app_new_name" in ids
|
||||
assert "app_old_name" not in ids, (
|
||||
"stale AST node for a re-extracted code file survived (#1116)"
|
||||
)
|
||||
assert len(after["nodes"]) < nodes_before, "polluted graph should shrink"
|
||||
|
||||
|
||||
# ── #2014: code-typed semantic nodes count as a doc's semantic layer ───────────
|
||||
|
||||
Reference in New Issue
Block a user