mirror of
https://github.com/safishamsi/graphify.git
synced 2026-07-18 21:37:34 +00:00
fix(watch): stop AST-quick-scanning docs that already have semantic nodes (#1915)
_rebuild_code fed Markdown/doc files (.md/.mdx/.qmd/.skill) into the AST quick-scan while _reconcile_existing_graph preserved the same docs' semantic (LLM) nodes, so every semantic-backed doc was represented twice (heading nodes + concept nodes) and the graph bloated ~4x vs the CLI `graphify . --update` path. Semantic now supersedes AST per doc source: before choosing extract_targets, read the existing graph's source identities that carry semantic doc nodes (non-_origin=="ast", gated on file_type=="document" so pre-#1865 marker-less graphs aren't misread) and exclude those docs from the quick-scan in both the full-rebuild and changed_paths branches. They stay in code_files, so the #1795 fail-closed corpus check and the shrink accounting still cover them — a previously-bloated graph self-heals on the next full rebuild (the AST ownership rule drops the stale heading nodes) without the shrink guard refusing the smaller write. On incremental rebuilds the excluded docs never enter rebuilt/node-evicted identities, mirroring #1865's tier-scoped edge rule at the node level, so a doc's semantic nodes and edges survive. Docs with no semantic layer (and brand-new docs) keep the no-LLM quick-scan structure from #09b33b7.
This commit is contained in:
safishamsi
+61
-2
@@ -876,16 +876,64 @@ def _rebuild_code(
|
||||
code_files = [Path(f) for f in detected['files']['code']]
|
||||
|
||||
# Include document files that have AST extractors (e.g. .md, .mdx, .qmd)
|
||||
ast_doc_files: list[Path] = []
|
||||
for doc_file in detected['files'].get('document', []):
|
||||
p = Path(doc_file)
|
||||
if _get_extractor(p) is not None:
|
||||
code_files.append(p)
|
||||
ast_doc_files.append(p)
|
||||
|
||||
existing_graph = out / "graph.json"
|
||||
if not code_files and not existing_graph.exists():
|
||||
print("[graphify watch] No code files found - nothing to rebuild.")
|
||||
return False
|
||||
|
||||
# #1915: a document that already carries SEMANTIC (LLM) nodes in the
|
||||
# existing graph must not ALSO be AST-quick-scanned — otherwise every
|
||||
# rebuild mints heading nodes on top of the preserved semantic nodes
|
||||
# and the doc is represented twice (~4x graph bloat vs the CLI update
|
||||
# path, which AST-extracts only code). Semantic supersedes AST per doc
|
||||
# source: the quick-scan stays as a fallback for docs with no semantic
|
||||
# layer (the no-LLM doc-structure feature, #09b33b7) and for brand-new
|
||||
# docs the graph has never seen. These docs stay in ``code_files`` so
|
||||
# corpus membership (#1795 fail-closed deletion evidence) and the
|
||||
# shrink accounting below still cover them — a previously-bloated
|
||||
# graph must be allowed to self-heal on a full rebuild without the
|
||||
# shrink-guard refusing the smaller write.
|
||||
semantic_doc_files: set[Path] = set()
|
||||
if ast_doc_files and existing_graph.exists():
|
||||
try:
|
||||
check_graph_file_size_cap(existing_graph)
|
||||
prior = json.loads(existing_graph.read_text(encoding="utf-8"))
|
||||
prior_paths = _StoredSourcePaths(
|
||||
prior,
|
||||
out=out,
|
||||
project_root=project_root,
|
||||
watch_root=watch_root,
|
||||
normalize_source=_nsf,
|
||||
)
|
||||
# Semantic doc nodes lack the AST origin marker. Gate on
|
||||
# file_type=="document" as well so a pre-#1865 graph whose
|
||||
# AST nodes lack the ``_origin`` marker isn't misread as
|
||||
# semantic-backed via some other marker-less node kind.
|
||||
semantic_doc_identities: set[str] = set()
|
||||
for node in prior.get("nodes", []):
|
||||
if node.get("_origin") == "ast":
|
||||
continue
|
||||
if node.get("file_type") != "document":
|
||||
continue
|
||||
identity = prior_paths.identity(node.get("source_file"))
|
||||
if identity:
|
||||
semantic_doc_identities.add(identity)
|
||||
if semantic_doc_identities:
|
||||
semantic_doc_files = {
|
||||
p for p in ast_doc_files
|
||||
if prior_paths.absolute_identity(str(p), project_root)
|
||||
in semantic_doc_identities
|
||||
}
|
||||
except Exception:
|
||||
semantic_doc_files = set()
|
||||
|
||||
# Incremental path: when the caller passed an explicit change list,
|
||||
# extract only changed-and-still-existing files. Deleted paths are
|
||||
# tracked separately so their stale nodes can be evicted below.
|
||||
@@ -898,6 +946,12 @@ def _rebuild_code(
|
||||
|
||||
if changed_paths is not None:
|
||||
code_set = {Path(os.path.abspath(p)) for p in code_files}
|
||||
# #1915: semantic-backed docs are never AST-quick-scanned; their
|
||||
# semantic nodes are the sole representation. Mirroring #1865's
|
||||
# tier-scoped edge rule at the node level, they also must NOT
|
||||
# enter extract_targets (hence rebuilt/node-evicted identities) on
|
||||
# an incremental rebuild, or their semantic nodes would be wiped.
|
||||
semantic_doc_set = {Path(os.path.abspath(p)) for p in semantic_doc_files}
|
||||
wanted: list[Path] = []
|
||||
change_root = Path.cwd().resolve()
|
||||
for raw in changed_paths:
|
||||
@@ -908,7 +962,7 @@ def _rebuild_code(
|
||||
)
|
||||
tracked = next((cand for cand in candidates if cand.exists() and cand in code_set), None)
|
||||
if tracked is not None:
|
||||
if tracked not in wanted:
|
||||
if tracked not in wanted and tracked not in semantic_doc_set:
|
||||
wanted.append(tracked)
|
||||
continue
|
||||
|
||||
@@ -938,7 +992,12 @@ def _rebuild_code(
|
||||
return True
|
||||
extract_targets = wanted
|
||||
else:
|
||||
extract_targets = code_files
|
||||
# Full rebuild: skip the AST quick-scan for semantic-backed docs
|
||||
# (#1915). They remain in code_files, so stale _origin=="ast"
|
||||
# heading nodes from a previously-bloated graph are dropped by the
|
||||
# full-rebuild AST ownership rule while the shrink accounting
|
||||
# below still counts the doc as a rebuilt source.
|
||||
extract_targets = [p for p in code_files if p not in semantic_doc_files]
|
||||
|
||||
commit = _git_head()
|
||||
result = extract(extract_targets, cache_root=watch_root) if extract_targets else {
|
||||
|
||||
@@ -1649,3 +1649,189 @@ def test_rebuild_code_still_evicts_when_excluded_file_is_also_deleted(tmp_path):
|
||||
labels = {n["label"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]}
|
||||
assert "brainstorm.md" not in labels, "deleted file's nodes must still be evicted"
|
||||
assert "login()" in labels
|
||||
|
||||
|
||||
# --- #1915: semantic-backed docs must not be double-represented by the AST quick-scan ---
|
||||
|
||||
|
||||
_SEMANTIC_GUIDE_IDS = {"guide_doc", "auth_flow", "session_model"}
|
||||
_AST_GUIDE_IDS = {"guide", "guide_overview", "guide_setup", "guide_usage"}
|
||||
|
||||
|
||||
def _seed_semantic_doc_graph(corpus):
|
||||
"""Build a code-only graph, then add guide.md represented ONLY semantically.
|
||||
|
||||
Mimics a graph produced by the CLI ``graphify . --update`` path: code AST
|
||||
nodes plus a semantic (LLM) layer for the document — a ``<slug>_doc`` node
|
||||
and concept nodes, none carrying the ``_origin`` marker — and NO AST
|
||||
heading nodes for the doc.
|
||||
"""
|
||||
from graphify.watch import _rebuild_code
|
||||
|
||||
corpus.mkdir()
|
||||
(corpus / "app.py").write_text(
|
||||
"def handle_login():\n return 1\n", encoding="utf-8"
|
||||
)
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
|
||||
(corpus / "guide.md").write_text(
|
||||
"# Overview\n\nIntro.\n\n## Setup\n\nSteps.\n\n## Usage\n\nMore.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
graph_path = corpus / "graphify-out" / "graph.json"
|
||||
data = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
code_node_id = next(
|
||||
n["id"] for n in data["nodes"] if n.get("source_file") == "app.py"
|
||||
)
|
||||
data["nodes"].extend([
|
||||
{"id": "guide_doc", "label": "Guide", "file_type": "document",
|
||||
"source_file": "guide.md"},
|
||||
{"id": "auth_flow", "label": "Auth Flow", "file_type": "concept",
|
||||
"source_file": "guide.md"},
|
||||
{"id": "session_model", "label": "Session Model", "file_type": "concept",
|
||||
"source_file": "guide.md"},
|
||||
])
|
||||
data["links"].extend([
|
||||
{"source": "guide_doc", "target": "auth_flow", "relation": "explains",
|
||||
"confidence": "INFERRED", "source_file": "guide.md"},
|
||||
{"source": "auth_flow", "target": code_node_id,
|
||||
"relation": "implemented_by", "confidence": "INFERRED",
|
||||
"source_file": "guide.md"},
|
||||
])
|
||||
graph_path.write_text(json.dumps(data), encoding="utf-8")
|
||||
return graph_path
|
||||
|
||||
|
||||
def test_rebuild_code_semantic_doc_not_double_represented_on_full_rebuild(tmp_path):
|
||||
"""#1915: a full _rebuild_code must not AST-quick-scan a doc whose semantic
|
||||
(LLM) nodes already represent it. Before the fix the quick-scan minted
|
||||
heading nodes ON TOP of the preserved semantic nodes, representing every
|
||||
doc twice (~4x bloated graph vs the CLI update path)."""
|
||||
from graphify.watch import _rebuild_code
|
||||
|
||||
corpus = tmp_path / "corpus"
|
||||
graph_path = _seed_semantic_doc_graph(corpus)
|
||||
before = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
|
||||
after = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
after_ids = {n["id"] for n in after["nodes"]}
|
||||
assert _SEMANTIC_GUIDE_IDS <= after_ids, "semantic doc nodes must be preserved"
|
||||
assert not (_AST_GUIDE_IDS & after_ids), (
|
||||
"AST heading nodes minted for a semantic-backed doc (#1915)"
|
||||
)
|
||||
assert len(after["nodes"]) == len(before["nodes"]), (
|
||||
f"node count inflated {len(before['nodes'])} -> {len(after['nodes'])} (#1915)"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"changed",
|
||||
[[Path("guide.md")], [Path("guide.md"), Path("app.py")]],
|
||||
ids=["doc-only", "doc-plus-code"],
|
||||
)
|
||||
def test_rebuild_code_incremental_preserves_semantic_doc_nodes_and_edges(
|
||||
tmp_path, changed
|
||||
):
|
||||
"""#1915: an incremental rebuild whose change set includes a semantic-backed
|
||||
doc must not wipe the doc's semantic nodes or their edges — re-extraction
|
||||
owns only a source's AST tier (node-level mirror of #1865's edge rule)."""
|
||||
from graphify.watch import _rebuild_code
|
||||
|
||||
corpus = tmp_path / "corpus"
|
||||
graph_path = _seed_semantic_doc_graph(corpus)
|
||||
|
||||
assert _rebuild_code(
|
||||
corpus, changed_paths=changed, no_cluster=True, acquire_lock=False
|
||||
) is True
|
||||
|
||||
after = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
after_ids = {n["id"] for n in after["nodes"]}
|
||||
assert _SEMANTIC_GUIDE_IDS <= after_ids, (
|
||||
"semantic doc nodes wiped by an incremental rebuild"
|
||||
)
|
||||
relations = {
|
||||
(e.get("source"), e.get("target"), e.get("relation"))
|
||||
for e in after["links"]
|
||||
}
|
||||
assert ("guide_doc", "auth_flow", "explains") in relations, (
|
||||
"semantic doc edge dropped by an incremental rebuild"
|
||||
)
|
||||
assert any(
|
||||
src == "auth_flow" and rel == "implemented_by"
|
||||
for src, _tgt, rel in relations
|
||||
), "doc-to-code semantic edge dropped by an incremental rebuild"
|
||||
assert not (_AST_GUIDE_IDS & after_ids), (
|
||||
"incremental rebuild AST-quick-scanned a semantic-backed doc (#1915)"
|
||||
)
|
||||
|
||||
|
||||
def test_rebuild_code_quick_scans_doc_without_semantic_nodes(tmp_path):
|
||||
"""#09b33b7 guard: a doc with NO semantic layer still gets the AST
|
||||
quick-scan so no-LLM corpora keep their heading structure — #1915's
|
||||
semantic-supersedes-AST rule must not regress the fallback."""
|
||||
from graphify.watch import _rebuild_code
|
||||
|
||||
corpus = tmp_path / "corpus"
|
||||
corpus.mkdir()
|
||||
(corpus / "app.py").write_text("def f():\n return 1\n", encoding="utf-8")
|
||||
(corpus / "notes.md").write_text("# Alpha\n\n## Beta\n", encoding="utf-8")
|
||||
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
graph_path = corpus / "graphify-out" / "graph.json"
|
||||
ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]}
|
||||
assert {"notes", "notes_alpha", "notes_beta"} <= ids
|
||||
|
||||
# A rebuild over the existing graph (still no semantic nodes for the doc)
|
||||
# keeps quick-scanning it rather than dropping its structure.
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]}
|
||||
assert {"notes", "notes_alpha", "notes_beta"} <= ids
|
||||
|
||||
|
||||
def test_rebuild_code_polluted_graph_self_heals_on_full_rebuild(tmp_path):
|
||||
"""#1915: a graph already bloated by the bug (semantic doc nodes PLUS stale
|
||||
_origin=="ast" heading nodes for the same doc) sheds the heading nodes on
|
||||
the next full rebuild via the AST ownership rule — and the shrink guard
|
||||
accepts the smaller write without --force."""
|
||||
from graphify.watch import _rebuild_code
|
||||
|
||||
corpus = tmp_path / "corpus"
|
||||
corpus.mkdir()
|
||||
(corpus / "app.py").write_text(
|
||||
"def handle_login():\n return 1\n", encoding="utf-8"
|
||||
)
|
||||
(corpus / "guide.md").write_text(
|
||||
"# Overview\n\n## Setup\n\n## Usage\n", encoding="utf-8"
|
||||
)
|
||||
# Initial build quick-scans guide.md (no semantic layer yet): AST nodes.
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
graph_path = corpus / "graphify-out" / "graph.json"
|
||||
data = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
assert _AST_GUIDE_IDS <= {n["id"] for n in data["nodes"]}
|
||||
|
||||
# Layer the semantic representation on top -> the double-represented state.
|
||||
data["nodes"].extend([
|
||||
{"id": "guide_doc", "label": "Guide", "file_type": "document",
|
||||
"source_file": "guide.md"},
|
||||
{"id": "auth_flow", "label": "Auth Flow", "file_type": "concept",
|
||||
"source_file": "guide.md"},
|
||||
])
|
||||
data["links"].append({
|
||||
"source": "guide_doc", "target": "auth_flow", "relation": "explains",
|
||||
"confidence": "INFERRED", "source_file": "guide.md",
|
||||
})
|
||||
graph_path.write_text(json.dumps(data), encoding="utf-8")
|
||||
nodes_before = len(data["nodes"])
|
||||
|
||||
# No force=True: the self-heal shrink must be accepted by the guard.
|
||||
assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True
|
||||
|
||||
after = json.loads(graph_path.read_text(encoding="utf-8"))
|
||||
after_ids = {n["id"] for n in after["nodes"]}
|
||||
assert {"guide_doc", "auth_flow"} <= after_ids
|
||||
assert not (_AST_GUIDE_IDS & after_ids), (
|
||||
"stale AST heading nodes for a semantic-backed doc must self-heal away"
|
||||
)
|
||||
assert len(after["nodes"]) < nodes_before, "polluted graph should shrink"
|
||||
|
||||
Reference in New Issue
Block a user