From 94189bac3ca7a3080f8bcc9beebe4d0bddd61ff9 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Mon, 20 Jul 2026 13:29:44 +0100 Subject: [PATCH] fix(watch): correct semantic-node eviction/preservation on update Three silent-data-loss fixes in the update/reconcile path: - #2051: a full `graphify update` now evicts semantic nodes whose non-AST source (a .txt/.pdf/.png with no code extractor) was deleted from disk. The corpus sweep only checked re-extractable files, so deleted docs'/images' LLM nodes survived as authoritative forever. Disk absence is now the deletion signal; remote/virtual sources (`://`) are left untouched. - #2056: an incremental rebuild whose change set names a present-but- unextractable file no longer treats it as a deletion (which evicted its semantic nodes and disabled the shrink guard). The guard now falls through to per-source accounting instead of a wholesale bypass on any deletion. - #2014: code-typed nodes the semantic pass surfaces from within a document now count as that doc's semantic layer, so a rebuild doesn't re-scan and drop them. Regression tests for each in tests/test_watch.py. --- graphify/watch.py | 68 ++++++++++++++++----- tests/test_watch.py | 145 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+), 14 deletions(-) diff --git a/graphify/watch.py b/graphify/watch.py index 37edc9cb..8b246314 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -495,11 +495,32 @@ def _reconcile_existing_graph( _alive_cache: dict[str, bool] = {} for node in existing.get("nodes", []): source_file = node.get("source_file") - if not source_file or _get_extractor(Path(source_file)) is None: - continue + if not source_file or "://" in source_file: + continue # sourceless stub or remote/virtual source: never evict identity = source_paths.identity(source_file) if not source_paths.in_watch_root(source_file): continue + if _get_extractor(Path(source_file)) is None: + # Non-AST source (semantic doc/paper/image — .txt/.pdf/.png/...): + # never present in current_sources (built from AST-extractable + # code_files), so corpus absence is meaningless. Disk absence is + # the ONLY deletion evidence here — otherwise its semantic nodes + # are preserved forever and returned as authoritative even after + # the file is deleted (#2051). A present-but-unextractable file + # stays preserved (alive -> skip). + if identity: + alive = _alive_cache.get(identity) + if alive is None: + alive = Path(identity).exists() + _alive_cache[identity] = alive + if not alive: + normalized = source_paths.normalize(source_file) + if normalized: + deleted_paths.add(normalized) + node_evicted_source_identities.add(identity) + edge_evicted_source_identities.add(identity) + hyperedge_evicted_source_identities.add(identity) + continue if identity not in current_sources: if identity: alive = _alive_cache.get(identity) @@ -721,7 +742,17 @@ def _check_shrink( plain ``graphify update`` after deleting a function refresh the graph without ``--force`` (#1116 left stale nodes write-blocked even though build dropped them). """ - if force or not existing_data or had_explicit_deletions: + if force or not existing_data: + return True + if had_explicit_deletions and rebuilt_sources is None: + # Legacy callers declare deletions but pass no rebuilt_sources, so the + # per-source accounting below can't run — keep the wholesale bypass for + # them. When rebuilt_sources IS given, deleted paths are folded into it + # (see call site), so genuine deletions still pass the _accounted check + # while an unexplained loss (a present-but-unextractable file wrongly + # routed to _add_deleted_source, or a dropped semantic node) is still + # caught rather than being waved through by the mere presence of any + # deletion in the change set (#2056). return True existing_nodes = existing_data.get("nodes", []) new_nodes = new_data.get("nodes", []) @@ -941,23 +972,24 @@ def _rebuild_code( ) # Semantic doc nodes lack the AST origin marker. Gate on the # doc-shaped subset of the six-value file_type enum - # (document/concept/rationale/paper, matching build.py's - # canonical set minus code/image) rather than "document" - # alone: per the extraction spec, a doc full of named + # (document/concept/rationale/paper AND code) rather than + # "document" alone: per the extraction spec, a doc full of named # concepts may be represented with ONLY concept/rationale # nodes and no separate "document" node — that's still # evidence of a semantic layer, not a marker-less AST node - # (#1954). The narrower pre-#1954 check under-recognized - # exactly that doc shape, letting it be re-quick-scanned - # every rebuild. A pre-#1865 graph whose AST nodes lack the - # ``_origin`` marker still isn't misread as semantic-backed, - # since "code" stays outside this set. + # (#1954). "code" is included too (#2014): the semantic pass + # legitimately mints code-typed nodes for symbols surfaced from + # WITHIN a doc (llm.py `_bind_node_evidence`), and it cannot be + # confused with a pre-#1865 marker-less AST code node — those are + # sourced from code files, which never intersect ast_doc_files + # below, whereas the AST quick-scan of a doc only ever mints + # "document" nodes (extractors/markdown.py). "image" stays out. semantic_doc_identities: set[str] = set() for node in prior.get("nodes", []): if node.get("_origin") == "ast": continue if node.get("file_type") not in ( - "document", "concept", "rationale", "paper" + "document", "concept", "rationale", "paper", "code" ): continue identity = prior_paths.identity(node.get("source_file")) @@ -1013,8 +1045,16 @@ def _rebuild_code( ) if existing_in_root is not None: # The path exists under the watched root but detect filtered - # it out. Evict any stale nodes that still claim it. - _add_deleted_source(existing_in_root) + # it out of code_set (no AST extractor, excluded, or + # sensitive). Existence is NOT deletion evidence (#2056): the + # file may carry semantic (LLM) nodes an AST rebuild cannot + # regenerate, and mis-routing it to _add_deleted_source both + # evicts those nodes AND sets had_explicit_deletions, which + # disables the shrink guard that would otherwise catch the + # loss. Preserve it — a genuine deletion still evicts via the + # branch below, the corpus sweep evicts a truly-gone non-AST + # source, and a deliberate exclusion is purged by a full + # re-extraction. continue deleted_in_root = next( diff --git a/tests/test_watch.py b/tests/test_watch.py index e665e773..cc0a1b39 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -1963,3 +1963,148 @@ def test_rebuild_code_polluted_graph_self_heals_on_full_rebuild(tmp_path): "stale AST heading nodes for a semantic-backed doc must self-heal away" ) assert len(after["nodes"]) < nodes_before, "polluted graph should shrink" + + +# ── #2014: code-typed semantic nodes count as a doc's semantic layer ─────────── + +_CODE_ONLY_GUIDE_IDS = {"parse_config", "load_settings"} + + +def _seed_semantic_doc_graph_code_only(corpus): + """Like ``_seed_semantic_doc_graph``, but guide.md's semantic layer is ONLY + code-typed nodes — symbols the LLM surfaced from WITHIN the doc (llm.py + ``_bind_node_evidence``), with no document/concept node at all (#2014).""" + from graphify.watch import _rebuild_code + + corpus.mkdir() + (corpus / "app.py").write_text( + "def handle_login():\n return 1\n", encoding="utf-8" + ) + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + (corpus / "guide.md").write_text( + "# Overview\n\nIntro.\n\n## Setup\n\nSteps.\n\n## Usage\n\nMore.\n", + encoding="utf-8", + ) + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + code_node_id = next( + n["id"] for n in data["nodes"] if n.get("source_file") == "app.py" + ) + data["nodes"].extend([ + {"id": "parse_config", "label": "parse_config()", "file_type": "code", + "source_file": "guide.md"}, + {"id": "load_settings", "label": "load_settings()", "file_type": "code", + "source_file": "guide.md"}, + ]) + data["links"].append({ + "source": "parse_config", "target": code_node_id, + "relation": "implemented_by", "confidence": "INFERRED", + "source_file": "guide.md", + }) + graph_path.write_text(json.dumps(data), encoding="utf-8") + return graph_path + + +def test_rebuild_code_code_only_semantic_doc_not_double_represented_on_full_rebuild( + tmp_path, +): + """#2014: a doc represented ONLY by code-typed semantic nodes (symbols + surfaced from within it) must be recognized as semantic-backed and skipped + by the AST quick-scan. Before the fix "code" was absent from the semantic + file_type gate, so the doc was re-AST-scanned — minting heading nodes AND + dropping the code-typed semantic nodes (they belonged to a now-rebuilt + source), silently losing them.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + graph_path = _seed_semantic_doc_graph_code_only(corpus) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + after_ids = {n["id"] for n in after["nodes"]} + assert _CODE_ONLY_GUIDE_IDS <= after_ids, ( + "code-typed semantic doc nodes dropped by a full rebuild (#2014)" + ) + assert not (_AST_GUIDE_IDS & after_ids), ( + "AST heading nodes minted for a code-only semantic-backed doc (#2014)" + ) + + +# ── #2051: deleted non-AST sources (docs/papers/images) get evicted ──────────── + +def test_rebuild_code_evicts_semantic_nodes_from_deleted_non_ast_source(tmp_path): + """#2051: a full `graphify update` must evict semantic nodes whose non-AST + source file (a .txt/.pdf/.png with no code extractor) was deleted from disk. + The corpus sweep used to skip every sourceless-of-extractor node, so those + nodes survived forever and were served as authoritative long after the file + was gone. Disk absence is the only deletion evidence for such sources.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text("def handle():\n return 1\n", encoding="utf-8") + # Two non-AST semantic sources: one stays on disk, one gets deleted. + (corpus / "kept.txt").write_text("Design rationale that stays.\n", encoding="utf-8") + (corpus / "gone.txt").write_text("Rationale that will be deleted.\n", encoding="utf-8") + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + # No LLM in tests, so inject the semantic layer these .txt files would carry. + data["nodes"].extend([ + {"id": "kept_concept", "label": "Kept Concept", "file_type": "concept", + "source_file": "kept.txt"}, + {"id": "gone_concept", "label": "Gone Concept", "file_type": "concept", + "source_file": "gone.txt"}, + ]) + graph_path.write_text(json.dumps(data), encoding="utf-8") + + # Delete one non-AST source; the other stays. + (corpus / "gone.txt").unlink() + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + after_ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert "gone_concept" not in after_ids, ( + "semantic node from a deleted non-AST source must be evicted (#2051)" + ) + assert "kept_concept" in after_ids, ( + "semantic node from a surviving non-AST source must be preserved" + ) + + +# ── #2056: present-but-unextractable files in a change set are not deletions ─── + +def test_rebuild_code_incremental_preserves_present_non_ast_source(tmp_path): + """#2056: an incremental rebuild whose change set names a file that exists but + has no AST extractor (a doc/paper/image, or an excluded path) must NOT treat + it as deleted. The old change-set loop routed any present-but-untracked file + to _add_deleted_source, evicting its semantic nodes AND flipping + had_explicit_deletions so the shrink guard waved the loss through.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text("def handle():\n return 1\n", encoding="utf-8") + (corpus / "spec.txt").write_text("A spec with a semantic layer.\n", encoding="utf-8") + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + data["nodes"].append( + {"id": "spec_concept", "label": "Spec Concept", "file_type": "concept", + "source_file": "spec.txt"} + ) + graph_path.write_text(json.dumps(data), encoding="utf-8") + + # spec.txt is present but not AST-extractable; app.py is a real code change. + assert _rebuild_code( + corpus, changed_paths=[Path("spec.txt"), Path("app.py")], + no_cluster=True, acquire_lock=False, + ) is True + + after_ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert "spec_concept" in after_ids, ( + "present-but-unextractable file in change set wrongly evicted as deleted (#2056)" + )