mirror of
https://github.com/safishamsi/graphify.git
synced 2026-09-09 23:25:47 +00:00
fix(watch): correct semantic-node eviction/preservation on update
Three silent-data-loss fixes in the update/reconcile path: - #2051: a full `graphify update` now evicts semantic nodes whose non-AST source (a .txt/.pdf/.png with no code extractor) was deleted from disk. The corpus sweep only checked re-extractable files, so deleted docs'/images' LLM nodes survived as authoritative forever. Disk absence is now the deletion signal; remote/virtual sources (`://`) are left untouched. - #2056: an incremental rebuild whose change set names a present-but- unextractable file no longer treats it as a deletion (which evicted its semantic nodes and disabled the shrink guard). The guard now falls through to per-source accounting instead of a wholesale bypass on any deletion. - #2014: code-typed nodes the semantic pass surfaces from within a document now count as that doc's semantic layer, so a rebuild doesn't re-scan and drop them. Regression tests for each in tests/test_watch.py.
This commit is contained in:
safishamsi
+54
-14
@@ -495,11 +495,32 @@ def _reconcile_existing_graph(
|
||||
_alive_cache: dict[str, bool] = {}
|
||||
for node in existing.get("nodes", []):
|
||||
source_file = node.get("source_file")
|
||||
if not source_file or _get_extractor(Path(source_file)) is None:
|
||||
continue
|
||||
if not source_file or "://" in source_file:
|
||||
continue # sourceless stub or remote/virtual source: never evict
|
||||
identity = source_paths.identity(source_file)
|
||||
if not source_paths.in_watch_root(source_file):
|
||||
continue
|
||||
if _get_extractor(Path(source_file)) is None:
|
||||
# Non-AST source (semantic doc/paper/image — .txt/.pdf/.png/...):
|
||||
# never present in current_sources (built from AST-extractable
|
||||
# code_files), so corpus absence is meaningless. Disk absence is
|
||||
# the ONLY deletion evidence here — otherwise its semantic nodes
|
||||
# are preserved forever and returned as authoritative even after
|
||||
# the file is deleted (#2051). A present-but-unextractable file
|
||||
# stays preserved (alive -> skip).
|
||||
if identity:
|
||||
alive = _alive_cache.get(identity)
|
||||
if alive is None:
|
||||
alive = Path(identity).exists()
|
||||
_alive_cache[identity] = alive
|
||||
if not alive:
|
||||
normalized = source_paths.normalize(source_file)
|
||||
if normalized:
|
||||
deleted_paths.add(normalized)
|
||||
node_evicted_source_identities.add(identity)
|
||||
edge_evicted_source_identities.add(identity)
|
||||
hyperedge_evicted_source_identities.add(identity)
|
||||
continue
|
||||
if identity not in current_sources:
|
||||
if identity:
|
||||
alive = _alive_cache.get(identity)
|
||||
@@ -721,7 +742,17 @@ def _check_shrink(
|
||||
plain ``graphify update`` after deleting a function refresh the graph without
|
||||
``--force`` (#1116 left stale nodes write-blocked even though build dropped them).
|
||||
"""
|
||||
if force or not existing_data or had_explicit_deletions:
|
||||
if force or not existing_data:
|
||||
return True
|
||||
if had_explicit_deletions and rebuilt_sources is None:
|
||||
# Legacy callers declare deletions but pass no rebuilt_sources, so the
|
||||
# per-source accounting below can't run — keep the wholesale bypass for
|
||||
# them. When rebuilt_sources IS given, deleted paths are folded into it
|
||||
# (see call site), so genuine deletions still pass the _accounted check
|
||||
# while an unexplained loss (a present-but-unextractable file wrongly
|
||||
# routed to _add_deleted_source, or a dropped semantic node) is still
|
||||
# caught rather than being waved through by the mere presence of any
|
||||
# deletion in the change set (#2056).
|
||||
return True
|
||||
existing_nodes = existing_data.get("nodes", [])
|
||||
new_nodes = new_data.get("nodes", [])
|
||||
@@ -941,23 +972,24 @@ def _rebuild_code(
|
||||
)
|
||||
# Semantic doc nodes lack the AST origin marker. Gate on the
|
||||
# doc-shaped subset of the six-value file_type enum
|
||||
# (document/concept/rationale/paper, matching build.py's
|
||||
# canonical set minus code/image) rather than "document"
|
||||
# alone: per the extraction spec, a doc full of named
|
||||
# (document/concept/rationale/paper AND code) rather than
|
||||
# "document" alone: per the extraction spec, a doc full of named
|
||||
# concepts may be represented with ONLY concept/rationale
|
||||
# nodes and no separate "document" node — that's still
|
||||
# evidence of a semantic layer, not a marker-less AST node
|
||||
# (#1954). The narrower pre-#1954 check under-recognized
|
||||
# exactly that doc shape, letting it be re-quick-scanned
|
||||
# every rebuild. A pre-#1865 graph whose AST nodes lack the
|
||||
# ``_origin`` marker still isn't misread as semantic-backed,
|
||||
# since "code" stays outside this set.
|
||||
# (#1954). "code" is included too (#2014): the semantic pass
|
||||
# legitimately mints code-typed nodes for symbols surfaced from
|
||||
# WITHIN a doc (llm.py `_bind_node_evidence`), and it cannot be
|
||||
# confused with a pre-#1865 marker-less AST code node — those are
|
||||
# sourced from code files, which never intersect ast_doc_files
|
||||
# below, whereas the AST quick-scan of a doc only ever mints
|
||||
# "document" nodes (extractors/markdown.py). "image" stays out.
|
||||
semantic_doc_identities: set[str] = set()
|
||||
for node in prior.get("nodes", []):
|
||||
if node.get("_origin") == "ast":
|
||||
continue
|
||||
if node.get("file_type") not in (
|
||||
"document", "concept", "rationale", "paper"
|
||||
"document", "concept", "rationale", "paper", "code"
|
||||
):
|
||||
continue
|
||||
identity = prior_paths.identity(node.get("source_file"))
|
||||
@@ -1013,8 +1045,16 @@ def _rebuild_code(
|
||||
)
|
||||
if existing_in_root is not None:
|
||||
# The path exists under the watched root but detect filtered
|
||||
# it out. Evict any stale nodes that still claim it.
|
||||
_add_deleted_source(existing_in_root)
|
||||
# it out of code_set (no AST extractor, excluded, or
|
||||
# sensitive). Existence is NOT deletion evidence (#2056): the
|
||||
# file may carry semantic (LLM) nodes an AST rebuild cannot
|
||||
# regenerate, and mis-routing it to _add_deleted_source both
|
||||
# evicts those nodes AND sets had_explicit_deletions, which
|
||||
# disables the shrink guard that would otherwise catch the
|
||||
# loss. Preserve it — a genuine deletion still evicts via the
|
||||
# branch below, the corpus sweep evicts a truly-gone non-AST
|
||||
# source, and a deliberate exclusion is purged by a full
|
||||
# re-extraction.
|
||||
continue
|
||||
|
||||
deleted_in_root = next(
|
||||
|
||||
@@ -1963,3 +1963,148 @@ def test_rebuild_code_polluted_graph_self_heals_on_full_rebuild(tmp_path):
|
||||
"stale AST heading nodes for a semantic-backed doc must self-heal away"
|
||||
)
|
||||
assert len(after["nodes"]) < nodes_before, "polluted graph should shrink"
|
||||
|
||||
|
||||
# ── #2014: code-typed semantic nodes count as a doc's semantic layer ───────────
|
||||
|
||||
_CODE_ONLY_GUIDE_IDS = {"parse_config", "load_settings"}
|
||||
|
||||
|
||||
def _seed_semantic_doc_graph_code_only(corpus):
|
||||
"""Like ``_seed_semantic_doc_graph``, but guide.md's semantic layer is ONLY
|
||||
code-typed nodes — symbols the LLM surfaced from WITHIN the doc (llm.py
|
||||
``_bind_node_evidence``), with no document/concept node at all (#2014)."""
|
||||
from graphify.watch import _rebuild_code
|
||||
|
||||
corpus.mkdir()
|
||||
(corpus / "app.py").write_text(
|
||||
"def handle_login():\n return 1\n", encoding="utf-8"
|
||||
)
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
|
||||
(corpus / "guide.md").write_text(
|
||||
"# Overview\n\nIntro.\n\n## Setup\n\nSteps.\n\n## Usage\n\nMore.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
graph_path = corpus / "graphify-out" / "graph.json"
|
||||
data = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
code_node_id = next(
|
||||
n["id"] for n in data["nodes"] if n.get("source_file") == "app.py"
|
||||
)
|
||||
data["nodes"].extend([
|
||||
{"id": "parse_config", "label": "parse_config()", "file_type": "code",
|
||||
"source_file": "guide.md"},
|
||||
{"id": "load_settings", "label": "load_settings()", "file_type": "code",
|
||||
"source_file": "guide.md"},
|
||||
])
|
||||
data["links"].append({
|
||||
"source": "parse_config", "target": code_node_id,
|
||||
"relation": "implemented_by", "confidence": "INFERRED",
|
||||
"source_file": "guide.md",
|
||||
})
|
||||
graph_path.write_text(json.dumps(data), encoding="utf-8")
|
||||
return graph_path
|
||||
|
||||
|
||||
def test_rebuild_code_code_only_semantic_doc_not_double_represented_on_full_rebuild(
|
||||
tmp_path,
|
||||
):
|
||||
"""#2014: a doc represented ONLY by code-typed semantic nodes (symbols
|
||||
surfaced from within it) must be recognized as semantic-backed and skipped
|
||||
by the AST quick-scan. Before the fix "code" was absent from the semantic
|
||||
file_type gate, so the doc was re-AST-scanned — minting heading nodes AND
|
||||
dropping the code-typed semantic nodes (they belonged to a now-rebuilt
|
||||
source), silently losing them."""
|
||||
from graphify.watch import _rebuild_code
|
||||
|
||||
corpus = tmp_path / "corpus"
|
||||
graph_path = _seed_semantic_doc_graph_code_only(corpus)
|
||||
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
|
||||
after = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
after_ids = {n["id"] for n in after["nodes"]}
|
||||
assert _CODE_ONLY_GUIDE_IDS <= after_ids, (
|
||||
"code-typed semantic doc nodes dropped by a full rebuild (#2014)"
|
||||
)
|
||||
assert not (_AST_GUIDE_IDS & after_ids), (
|
||||
"AST heading nodes minted for a code-only semantic-backed doc (#2014)"
|
||||
)
|
||||
|
||||
|
||||
# ── #2051: deleted non-AST sources (docs/papers/images) get evicted ────────────
|
||||
|
||||
def test_rebuild_code_evicts_semantic_nodes_from_deleted_non_ast_source(tmp_path):
|
||||
"""#2051: a full `graphify update` must evict semantic nodes whose non-AST
|
||||
source file (a .txt/.pdf/.png with no code extractor) was deleted from disk.
|
||||
The corpus sweep used to skip every sourceless-of-extractor node, so those
|
||||
nodes survived forever and were served as authoritative long after the file
|
||||
was gone. Disk absence is the only deletion evidence for such sources."""
|
||||
from graphify.watch import _rebuild_code
|
||||
|
||||
corpus = tmp_path / "corpus"
|
||||
corpus.mkdir()
|
||||
(corpus / "app.py").write_text("def handle():\n return 1\n", encoding="utf-8")
|
||||
# Two non-AST semantic sources: one stays on disk, one gets deleted.
|
||||
(corpus / "kept.txt").write_text("Design rationale that stays.\n", encoding="utf-8")
|
||||
(corpus / "gone.txt").write_text("Rationale that will be deleted.\n", encoding="utf-8")
|
||||
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
graph_path = corpus / "graphify-out" / "graph.json"
|
||||
data = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
# No LLM in tests, so inject the semantic layer these .txt files would carry.
|
||||
data["nodes"].extend([
|
||||
{"id": "kept_concept", "label": "Kept Concept", "file_type": "concept",
|
||||
"source_file": "kept.txt"},
|
||||
{"id": "gone_concept", "label": "Gone Concept", "file_type": "concept",
|
||||
"source_file": "gone.txt"},
|
||||
])
|
||||
graph_path.write_text(json.dumps(data), encoding="utf-8")
|
||||
|
||||
# Delete one non-AST source; the other stays.
|
||||
(corpus / "gone.txt").unlink()
|
||||
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
after_ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]}
|
||||
assert "gone_concept" not in after_ids, (
|
||||
"semantic node from a deleted non-AST source must be evicted (#2051)"
|
||||
)
|
||||
assert "kept_concept" in after_ids, (
|
||||
"semantic node from a surviving non-AST source must be preserved"
|
||||
)
|
||||
|
||||
|
||||
# ── #2056: present-but-unextractable files in a change set are not deletions ───
|
||||
|
||||
def test_rebuild_code_incremental_preserves_present_non_ast_source(tmp_path):
|
||||
"""#2056: an incremental rebuild whose change set names a file that exists but
|
||||
has no AST extractor (a doc/paper/image, or an excluded path) must NOT treat
|
||||
it as deleted. The old change-set loop routed any present-but-untracked file
|
||||
to _add_deleted_source, evicting its semantic nodes AND flipping
|
||||
had_explicit_deletions so the shrink guard waved the loss through."""
|
||||
from graphify.watch import _rebuild_code
|
||||
|
||||
corpus = tmp_path / "corpus"
|
||||
corpus.mkdir()
|
||||
(corpus / "app.py").write_text("def handle():\n return 1\n", encoding="utf-8")
|
||||
(corpus / "spec.txt").write_text("A spec with a semantic layer.\n", encoding="utf-8")
|
||||
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
graph_path = corpus / "graphify-out" / "graph.json"
|
||||
data = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
data["nodes"].append(
|
||||
{"id": "spec_concept", "label": "Spec Concept", "file_type": "concept",
|
||||
"source_file": "spec.txt"}
|
||||
)
|
||||
graph_path.write_text(json.dumps(data), encoding="utf-8")
|
||||
|
||||
# spec.txt is present but not AST-extractable; app.py is a real code change.
|
||||
assert _rebuild_code(
|
||||
corpus, changed_paths=[Path("spec.txt"), Path("app.py")],
|
||||
no_cluster=True, acquire_lock=False,
|
||||
) is True
|
||||
|
||||
after_ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]}
|
||||
assert "spec_concept" in after_ids, (
|
||||
"present-but-unextractable file in change set wrongly evicted as deleted (#2056)"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user