diff --git a/graphify/build.py b/graphify/build.py index 151e65bb4..103eba3a4 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -53,6 +53,21 @@ def _is_ast_tier(item: dict) -> bool: return isinstance(loc, str) and bool(_AST_LOC_RE.match(loc)) +def _backfill_origin(item: dict) -> None: + """Stamp a loaded legacy item's tier via the _is_ast_tier shape fallback (#2334). + + External import stubs are skipped, and a stamp an earlier run gave one is + removed: a stub is re-minted from its edges on every write, so a stamped + copy differed from a fresh build and, read as ``semantic``, was kept after + its last import edge was gone. Unstamped, every tier decision reads it + exactly as before (the shape fallback says non-AST). + """ + if _is_external_stub(item): + item.pop("_origin", None) + return + item.setdefault("_origin", "ast" if _is_ast_tier(item) else "semantic") + + # Relations that say only "these two symbols appear together", with no claim about # HOW. An extractor that finds a specific fact for a pair — a call, an import, an # inheritance — routinely emits one of these for the same pair as well, so when the @@ -103,6 +118,21 @@ def _mint_external_stub(G: "nx.Graph", node_set: set, nid: str) -> None: node_set.add(nid) +def _is_external_stub(item: dict) -> bool: + """True for a node minted by :func:`_mint_external_stub` / :func:`mint_external_stubs_in_data`. + + A stub is derived from the import edges that point at it and is minted again + on every write, so it belongs to no extraction tier: the tier backfill must + not stamp it (a ``semantic`` stamp made it outlive its last import edge), and + a rebuild drops it once no edge references it. + """ + return ( + item.get("external") is True + and item.get("type") == "external" + and not item.get("source_file") + ) + + def mint_external_stubs_in_data(data: dict) -> None: """Mint external stub nodes for import-family links with an undeclared target. @@ -1726,12 +1756,9 @@ def _load_existing_graph(graph_path: Path) -> "tuple[list, list, list, bool] | N # unstamped items. Stamp them via the _is_ast_tier shape fallback so the # graph self-heals on the next write and every downstream tier decision # (build_merge replace, watch reconcile) reads an explicit marker. - for item in nodes: + for item in nodes + edges: if isinstance(item, dict): - item.setdefault("_origin", "ast" if _is_ast_tier(item) else "semantic") - for item in edges: - if isinstance(item, dict): - item.setdefault("_origin", "ast" if _is_ast_tier(item) else "semantic") + _backfill_origin(item) return ( nodes, edges, diff --git a/graphify/watch.py b/graphify/watch.py index ef6bb6db7..a3be7d0b0 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -957,11 +957,11 @@ def _reconcile_existing_graph( # build._load_existing_graph (this reconcile path loads the raw dict # separately, so the backfill there does not reach it). Stamping preserved # items means the graph self-heals on this write. - from graphify.build import _is_ast_tier + from graphify.build import _backfill_origin, _is_ast_tier, _is_external_stub for _bucket in ("nodes", "links", "edges"): for _item in existing.get(_bucket, []): if isinstance(_item, dict): - _item.setdefault("_origin", "ast" if _is_ast_tier(_item) else "semantic") + _backfill_origin(_item) try: from graphify.build import _norm_source_file as _nsf @@ -1202,6 +1202,25 @@ def _reconcile_existing_graph( continue preserved_hyperedges.append(edge) + # An external import stub exists only so an import edge has a declared + # endpoint (#2873); every write mints the ones still needed. Keeping one + # whose last edge is gone left a permanent zero-degree node that a fresh + # build of the same tree does not have. + referenced_ids = { + endpoint + for edge in result["edges"] + preserved_edges + for endpoint in (edge.get("source"), edge.get("target")) + } + for edge in preserved_hyperedges + result.get("hyperedges", []): + members = edge.get("nodes", edge.get("members", edge.get("node_ids", []))) + if isinstance(members, list): + referenced_ids.update(members) + preserved_nodes = [ + node + for node in preserved_nodes + if not _is_external_stub(node) or node["id"] in referenced_ids + ] + for item in preserved_nodes + preserved_edges + preserved_hyperedges: source_paths.rebase_preserved(item) diff --git a/tests/test_external_stub_endpoints.py b/tests/test_external_stub_endpoints.py index 1160c399a..4e92d1555 100644 --- a/tests/test_external_stub_endpoints.py +++ b/tests/test_external_stub_endpoints.py @@ -16,6 +16,7 @@ import json from pathlib import Path import networkx as nx +import pytest from graphify.build import ( build_from_json, @@ -129,3 +130,109 @@ def test_no_cluster_update_leaves_no_undeclared_endpoint(tmp_path: Path): # the stdlib imports resolved to typed external stubs externals = {n["id"] for n in data["nodes"] if n.get("external")} assert {"typing", "pathlib", "re"} <= externals + + +def _graph(corpus: Path) -> dict: + return json.loads((corpus / "graphify-out" / "graph.json").read_text(encoding="utf-8")) + + +def _two_file_corpus(tmp_path: Path) -> Path: + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "a.py").write_text( + "import bisect\nimport json\n\n\ndef find(xs, x):\n return bisect.bisect_left(xs, x)\n", + encoding="utf-8", + ) + (corpus / "b.py").write_text( + "import json\nfrom a import find\n\n\ndef use():\n return json.dumps(find([1, 2], 2))\n", + encoding="utf-8", + ) + return corpus + + +@pytest.mark.parametrize("no_cluster", [True, False], ids=["no_cluster", "clustered"]) +@pytest.mark.parametrize("incremental", [False, True], ids=["full", "incremental"]) +def test_external_stub_is_dropped_with_its_last_import(tmp_path: Path, no_cluster: bool, incremental: bool): + """A stub outlived its last import edge: the reload backfill stamped it + `_origin=semantic`, reconcile keeps semantic nodes, and the node stayed in + graph.json forever with no edge — a node a fresh build does not have.""" + from graphify.watch import _rebuild_code + + corpus = _two_file_corpus(tmp_path) + assert _rebuild_code(corpus, no_cluster=no_cluster, acquire_lock=False) is True + assert _rebuild_code(corpus, no_cluster=no_cluster, acquire_lock=False) is True + assert "bisect" in {n["id"] for n in _graph(corpus)["nodes"]} + + (corpus / "a.py").write_text( + "import json\n\n\ndef find(xs, x):\n return xs.index(x)\n", encoding="utf-8" + ) + changed = [corpus / "a.py"] if incremental else None + assert _rebuild_code(corpus, changed_paths=changed, no_cluster=no_cluster, acquire_lock=False) is True + + data = _graph(corpus) + ids = {n["id"] for n in data["nodes"]} + assert "bisect" not in ids, "stub of a removed import must not outlive its last edge" + # a stub other import edges still point at is kept + assert "json" in ids + links = data.get("links", data.get("edges", [])) + assert any(e["target"] == "json" for e in links) + + +def test_rebuild_writes_external_stubs_like_a_fresh_build(tmp_path: Path): + """Rebuilding an unchanged tree gave external stubs `_origin: semantic` + that the first build never wrote, so the same tree had two graph.json files.""" + from graphify.watch import _rebuild_code + + corpus = _two_file_corpus(tmp_path) + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + first = {n["id"]: n for n in _graph(corpus)["nodes"]} + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + second = {n["id"]: n for n in _graph(corpus)["nodes"]} + assert first["bisect"] == second["bisect"] + assert first["json"] == second["json"] + assert "_origin" not in second["json"] + + +def test_rebuild_heals_stub_left_by_an_earlier_version(tmp_path: Path): + """A graph written before the fix carries `_origin: semantic` stubs, some + with no edge left. One rebuild drops the orphan and clears the stamp.""" + from graphify.watch import _rebuild_code + + corpus = _two_file_corpus(tmp_path) + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + data = _graph(corpus) + for node in data["nodes"]: + if node.get("external"): + node["_origin"] = "semantic" + data["nodes"].append({ + "id": "requests", "label": "requests", "file_type": "concept", "type": "external", + "external": True, "source_file": "", "_origin": "semantic", + }) + graph_path.write_text(json.dumps(data), encoding="utf-8") + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + nodes = {n["id"]: n for n in _graph(corpus)["nodes"]} + assert "requests" not in nodes + assert "_origin" not in nodes["bisect"] + + +def test_backfill_origin_leaves_external_stubs_unstamped(): + from graphify.build import _backfill_origin + + stub = {"id": "os", "label": "os", "file_type": "concept", "type": "external", + "external": True, "source_file": "", "_origin": "semantic"} + _backfill_origin(stub) + assert "_origin" not in stub + + legacy_ast = {"id": "a_f", "source_file": "a.py", "source_location": "L3"} + _backfill_origin(legacy_ast) + assert legacy_ast["_origin"] == "ast" + + legacy_semantic = {"id": "concept_x", "source_file": "notes.md", "source_location": None} + _backfill_origin(legacy_semantic) + assert legacy_semantic["_origin"] == "semantic" + + stamped = {"id": "b_g", "source_file": "b.py", "_origin": "semantic", "external": True} + _backfill_origin(stamped) # not a stub (has a source file): an explicit stamp wins + assert stamped["_origin"] == "semantic"