diff --git a/graphify/watch.py b/graphify/watch.py index 57364228..a4eee588 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -876,16 +876,64 @@ def _rebuild_code( code_files = [Path(f) for f in detected['files']['code']] # Include document files that have AST extractors (e.g. .md, .mdx, .qmd) + ast_doc_files: list[Path] = [] for doc_file in detected['files'].get('document', []): p = Path(doc_file) if _get_extractor(p) is not None: code_files.append(p) + ast_doc_files.append(p) existing_graph = out / "graph.json" if not code_files and not existing_graph.exists(): print("[graphify watch] No code files found - nothing to rebuild.") return False + # #1915: a document that already carries SEMANTIC (LLM) nodes in the + # existing graph must not ALSO be AST-quick-scanned — otherwise every + # rebuild mints heading nodes on top of the preserved semantic nodes + # and the doc is represented twice (~4x graph bloat vs the CLI update + # path, which AST-extracts only code). Semantic supersedes AST per doc + # source: the quick-scan stays as a fallback for docs with no semantic + # layer (the no-LLM doc-structure feature, #09b33b7) and for brand-new + # docs the graph has never seen. These docs stay in ``code_files`` so + # corpus membership (#1795 fail-closed deletion evidence) and the + # shrink accounting below still cover them — a previously-bloated + # graph must be allowed to self-heal on a full rebuild without the + # shrink-guard refusing the smaller write. + semantic_doc_files: set[Path] = set() + if ast_doc_files and existing_graph.exists(): + try: + check_graph_file_size_cap(existing_graph) + prior = json.loads(existing_graph.read_text(encoding="utf-8")) + prior_paths = _StoredSourcePaths( + prior, + out=out, + project_root=project_root, + watch_root=watch_root, + normalize_source=_nsf, + ) + # Semantic doc nodes lack the AST origin marker. Gate on + # file_type=="document" as well so a pre-#1865 graph whose + # AST nodes lack the ``_origin`` marker isn't misread as + # semantic-backed via some other marker-less node kind. + semantic_doc_identities: set[str] = set() + for node in prior.get("nodes", []): + if node.get("_origin") == "ast": + continue + if node.get("file_type") != "document": + continue + identity = prior_paths.identity(node.get("source_file")) + if identity: + semantic_doc_identities.add(identity) + if semantic_doc_identities: + semantic_doc_files = { + p for p in ast_doc_files + if prior_paths.absolute_identity(str(p), project_root) + in semantic_doc_identities + } + except Exception: + semantic_doc_files = set() + # Incremental path: when the caller passed an explicit change list, # extract only changed-and-still-existing files. Deleted paths are # tracked separately so their stale nodes can be evicted below. @@ -898,6 +946,12 @@ def _rebuild_code( if changed_paths is not None: code_set = {Path(os.path.abspath(p)) for p in code_files} + # #1915: semantic-backed docs are never AST-quick-scanned; their + # semantic nodes are the sole representation. Mirroring #1865's + # tier-scoped edge rule at the node level, they also must NOT + # enter extract_targets (hence rebuilt/node-evicted identities) on + # an incremental rebuild, or their semantic nodes would be wiped. + semantic_doc_set = {Path(os.path.abspath(p)) for p in semantic_doc_files} wanted: list[Path] = [] change_root = Path.cwd().resolve() for raw in changed_paths: @@ -908,7 +962,7 @@ def _rebuild_code( ) tracked = next((cand for cand in candidates if cand.exists() and cand in code_set), None) if tracked is not None: - if tracked not in wanted: + if tracked not in wanted and tracked not in semantic_doc_set: wanted.append(tracked) continue @@ -938,7 +992,12 @@ def _rebuild_code( return True extract_targets = wanted else: - extract_targets = code_files + # Full rebuild: skip the AST quick-scan for semantic-backed docs + # (#1915). They remain in code_files, so stale _origin=="ast" + # heading nodes from a previously-bloated graph are dropped by the + # full-rebuild AST ownership rule while the shrink accounting + # below still counts the doc as a rebuilt source. + extract_targets = [p for p in code_files if p not in semantic_doc_files] commit = _git_head() result = extract(extract_targets, cache_root=watch_root) if extract_targets else { diff --git a/tests/test_watch.py b/tests/test_watch.py index 8df9c1b8..fc91cf18 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -1649,3 +1649,189 @@ def test_rebuild_code_still_evicts_when_excluded_file_is_also_deleted(tmp_path): labels = {n["label"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} assert "brainstorm.md" not in labels, "deleted file's nodes must still be evicted" assert "login()" in labels + + +# --- #1915: semantic-backed docs must not be double-represented by the AST quick-scan --- + + +_SEMANTIC_GUIDE_IDS = {"guide_doc", "auth_flow", "session_model"} +_AST_GUIDE_IDS = {"guide", "guide_overview", "guide_setup", "guide_usage"} + + +def _seed_semantic_doc_graph(corpus): + """Build a code-only graph, then add guide.md represented ONLY semantically. + + Mimics a graph produced by the CLI ``graphify . --update`` path: code AST + nodes plus a semantic (LLM) layer for the document — a ``_doc`` node + and concept nodes, none carrying the ``_origin`` marker — and NO AST + heading nodes for the doc. + """ + from graphify.watch import _rebuild_code + + corpus.mkdir() + (corpus / "app.py").write_text( + "def handle_login():\n return 1\n", encoding="utf-8" + ) + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + (corpus / "guide.md").write_text( + "# Overview\n\nIntro.\n\n## Setup\n\nSteps.\n\n## Usage\n\nMore.\n", + encoding="utf-8", + ) + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + code_node_id = next( + n["id"] for n in data["nodes"] if n.get("source_file") == "app.py" + ) + data["nodes"].extend([ + {"id": "guide_doc", "label": "Guide", "file_type": "document", + "source_file": "guide.md"}, + {"id": "auth_flow", "label": "Auth Flow", "file_type": "concept", + "source_file": "guide.md"}, + {"id": "session_model", "label": "Session Model", "file_type": "concept", + "source_file": "guide.md"}, + ]) + data["links"].extend([ + {"source": "guide_doc", "target": "auth_flow", "relation": "explains", + "confidence": "INFERRED", "source_file": "guide.md"}, + {"source": "auth_flow", "target": code_node_id, + "relation": "implemented_by", "confidence": "INFERRED", + "source_file": "guide.md"}, + ]) + graph_path.write_text(json.dumps(data), encoding="utf-8") + return graph_path + + +def test_rebuild_code_semantic_doc_not_double_represented_on_full_rebuild(tmp_path): + """#1915: a full _rebuild_code must not AST-quick-scan a doc whose semantic + (LLM) nodes already represent it. Before the fix the quick-scan minted + heading nodes ON TOP of the preserved semantic nodes, representing every + doc twice (~4x bloated graph vs the CLI update path).""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + graph_path = _seed_semantic_doc_graph(corpus) + before = json.loads(graph_path.read_text(encoding="utf-8")) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + after_ids = {n["id"] for n in after["nodes"]} + assert _SEMANTIC_GUIDE_IDS <= after_ids, "semantic doc nodes must be preserved" + assert not (_AST_GUIDE_IDS & after_ids), ( + "AST heading nodes minted for a semantic-backed doc (#1915)" + ) + assert len(after["nodes"]) == len(before["nodes"]), ( + f"node count inflated {len(before['nodes'])} -> {len(after['nodes'])} (#1915)" + ) + + +@pytest.mark.parametrize( + "changed", + [[Path("guide.md")], [Path("guide.md"), Path("app.py")]], + ids=["doc-only", "doc-plus-code"], +) +def test_rebuild_code_incremental_preserves_semantic_doc_nodes_and_edges( + tmp_path, changed +): + """#1915: an incremental rebuild whose change set includes a semantic-backed + doc must not wipe the doc's semantic nodes or their edges — re-extraction + owns only a source's AST tier (node-level mirror of #1865's edge rule).""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + graph_path = _seed_semantic_doc_graph(corpus) + + assert _rebuild_code( + corpus, changed_paths=changed, no_cluster=True, acquire_lock=False + ) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + after_ids = {n["id"] for n in after["nodes"]} + assert _SEMANTIC_GUIDE_IDS <= after_ids, ( + "semantic doc nodes wiped by an incremental rebuild" + ) + relations = { + (e.get("source"), e.get("target"), e.get("relation")) + for e in after["links"] + } + assert ("guide_doc", "auth_flow", "explains") in relations, ( + "semantic doc edge dropped by an incremental rebuild" + ) + assert any( + src == "auth_flow" and rel == "implemented_by" + for src, _tgt, rel in relations + ), "doc-to-code semantic edge dropped by an incremental rebuild" + assert not (_AST_GUIDE_IDS & after_ids), ( + "incremental rebuild AST-quick-scanned a semantic-backed doc (#1915)" + ) + + +def test_rebuild_code_quick_scans_doc_without_semantic_nodes(tmp_path): + """#09b33b7 guard: a doc with NO semantic layer still gets the AST + quick-scan so no-LLM corpora keep their heading structure — #1915's + semantic-supersedes-AST rule must not regress the fallback.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text("def f():\n return 1\n", encoding="utf-8") + (corpus / "notes.md").write_text("# Alpha\n\n## Beta\n", encoding="utf-8") + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert {"notes", "notes_alpha", "notes_beta"} <= ids + + # A rebuild over the existing graph (still no semantic nodes for the doc) + # keeps quick-scanning it rather than dropping its structure. + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert {"notes", "notes_alpha", "notes_beta"} <= ids + + +def test_rebuild_code_polluted_graph_self_heals_on_full_rebuild(tmp_path): + """#1915: a graph already bloated by the bug (semantic doc nodes PLUS stale + _origin=="ast" heading nodes for the same doc) sheds the heading nodes on + the next full rebuild via the AST ownership rule — and the shrink guard + accepts the smaller write without --force.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text( + "def handle_login():\n return 1\n", encoding="utf-8" + ) + (corpus / "guide.md").write_text( + "# Overview\n\n## Setup\n\n## Usage\n", encoding="utf-8" + ) + # Initial build quick-scans guide.md (no semantic layer yet): AST nodes. + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + assert _AST_GUIDE_IDS <= {n["id"] for n in data["nodes"]} + + # Layer the semantic representation on top -> the double-represented state. + data["nodes"].extend([ + {"id": "guide_doc", "label": "Guide", "file_type": "document", + "source_file": "guide.md"}, + {"id": "auth_flow", "label": "Auth Flow", "file_type": "concept", + "source_file": "guide.md"}, + ]) + data["links"].append({ + "source": "guide_doc", "target": "auth_flow", "relation": "explains", + "confidence": "INFERRED", "source_file": "guide.md", + }) + graph_path.write_text(json.dumps(data), encoding="utf-8") + nodes_before = len(data["nodes"]) + + # No force=True: the self-heal shrink must be accepted by the guard. + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + after_ids = {n["id"] for n in after["nodes"]} + assert {"guide_doc", "auth_flow"} <= after_ids + assert not (_AST_GUIDE_IDS & after_ids), ( + "stale AST heading nodes for a semantic-backed doc must self-heal away" + ) + assert len(after["nodes"]) < nodes_before, "polluted graph should shrink"