From c68d167ee4b55a65c1955b31eb90b2589eedcdbb Mon Sep 17 00:00:00 2001 From: Krish Garg <181051779+krishhgg@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:13:07 +0000 Subject: [PATCH] fix(rust): expand grouped use declarations into import bindings Give each binding of a use tree the import edge that v8 produces for its equivalent plain use declaration. Preserve aliases, path comments, separator whitespace, nested prefixes and self/glob binding behavior. Keep the old group edge as call evidence until the call pass finishes; exclude per-binding edges from that pass, then remove private markers. Plain imports and all call edges retain their current-v8 behavior. AST cache schema 6 -> 7 rejects results written before expansion. Replace the earlier broad resolver in #4135 with this syntax-only fix. The resolver is preserved on local branch wip/rust-use-resolver. On current v8, all 30 new tests and the full 6642-test suite pass. Fresh ripgrep builds have 864 import edges instead of 247, with every call unchanged. Existing name-based import rewiring can still produce wrong matches, just as it does for equivalent plain declarations. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Codex (cherry picked from commit 2669ee4f1178a2ea66dd976a431a821cef896d99) --- graphify/cache.py | 2 +- graphify/extract.py | 7 +- graphify/extractors/rust.py | 43 ++++++- tests/test_rust_use_trees.py | 209 +++++++++++++++++++++++++++++++++++ 4 files changed, 256 insertions(+), 5 deletions(-) create mode 100644 tests/test_rust_use_trees.py diff --git a/graphify/cache.py b/graphify/cache.py index 1f62efbca..f4cddebc0 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -37,7 +37,7 @@ except Exception: _EXTRACTOR_VERSION = "unknown" # Bump when AST cache-key semantics change independently of the package version. -_AST_CACHE_SCHEMA = 7 # Python opaque-base/super(args) raw-call markers and nested-class enclosing ids. +_AST_CACHE_SCHEMA = 8 # Rust `use` groups: one imports_from edge per binding; prior (7): Python opaque-base/super(args) raw-call markers and nested-class enclosing ids. # Per-process memo: distributions() scans site-packages metadata, which is diff --git a/graphify/extract.py b/graphify/extract.py index ae924504f..6343c294d 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -8980,7 +8980,7 @@ def extract( for e in all_edges: if e.get("relation") == "imports": file_to_symbol_imports.setdefault(e["source"], set()).add(e["target"]) - elif e.get("relation") == "imports_from": + elif e.get("relation") == "imports_from" and not e.get("_rust_use_binding"): file_to_module_imports.setdefault(e["source"], set()).add(e["target"]) # Map each node back to its containing file node id so we can ask @@ -9383,6 +9383,11 @@ def extract( else: run_language_resolvers(paths, per_file, all_nodes, all_edges) + # A Rust `use` group keeps one edge per binding; its v8 edge only served the calls above. + all_edges[:] = [e for e in all_edges if not e.pop("_rust_use_group", False)] + for e in all_edges: + e.pop("_rust_use_binding", None) + # Relativize source_file fields so paths are portable across machines (#555). # When the node's id was itself minted from the absolute path, remap it to a # portable id and rewrite the edge endpoints that reference it. diff --git a/graphify/extractors/rust.py b/graphify/extractors/rust.py index 452758050..e99f63410 100644 --- a/graphify/extractors/rust.py +++ b/graphify/extractors/rust.py @@ -153,6 +153,37 @@ def _rust_simple_generic_impl_key(node, source: bytes) -> str | None: return f"{owner_name}/{len(parameter_names)}" if owner_name else None +def _rust_use_module(argument: str) -> str: + """The name a plain `use` imports from: the last path segment of its argument, as + written (`a::b` -> `b`, `a::b as c` -> `b as c`, `a::*` -> `a`).""" + clean = argument.split("{")[0].rstrip(":").rstrip("*").rstrip(":") + return clean.split("::")[-1].strip() + + +def _rust_use_bindings(node, source: bytes, prefix: str = "", self_path: str = "") -> list[tuple[str, int]]: + """Each binding of a `use` tree, as the argument of a plain `use` that binds the same + name, with its line: `a::{b, c as d, e::{self}, f::*}` gives `b`, `c as d`, `a::e` + and `f::*` (as `use a::b;` and so on would: only the last segment counts). Paths + keep their text as written.""" + t, line = node.type, node.start_point[0] + 1 + if t == "use_list": + return [binding for child in node.named_children for binding in _rust_use_bindings(child, source, prefix, self_path)] + if t == "scoped_use_list": + path, group = node.child_by_field_name("path"), node.child_by_field_name("list") + lead = prefix + source[node.start_byte:group.start_byte].decode("utf-8", errors="replace") if group is not None else "" + owner = prefix + _read_text(path, source) if path is not None else "" + return _rust_use_bindings(group, source, lead, owner) if group is not None else [] + alias = "" + if t == "use_as_clause": + clause, node = node, node.child_by_field_name("path") + alias = source[node.end_byte:clause.end_byte].decode("utf-8", errors="replace") if node is not None else "" + if node is not None and node.type == "self" and self_path: # `a::{self}` binds `a` + return [(self_path + alias, line)] + if node is not None and node.type in ("use_wildcard", "identifier", "scoped_identifier", "crate", "self", "super"): + return [(prefix + _read_text(node, source) + alias, line)] + return [] + + _RUST_TRAIT_METHOD_BLOCKLIST: frozenset[str] = frozenset({ "new", "default", "parse", "from_str", "now", "clone", "into", "from", "to_string", "to_owned", "len", "is_empty", "iter", "next", "build", @@ -690,12 +721,18 @@ def extract_rust(path: Path) -> dict: if t == "use_declaration": arg = node.child_by_field_name("argument") if arg: - raw = _read_text(arg, source) - clean = raw.split("{")[0].rstrip(":").rstrip("*").rstrip(":") - module_name = clean.split("::")[-1].strip() + module_name = _rust_use_module(_read_text(arg, source)) + grouped = arg.type in ("use_list", "scoped_use_list") if module_name: tgt_nid = _make_id(module_name) add_edge(file_nid, tgt_nid, "imports_from", node.start_point[0] + 1, context="import") + if grouped: # call evidence as in v8, then dropped (extract.py) + edges[-1]["_rust_use_group"] = True + # A group imports what the plain `use` of each of its bindings would. + for binding, line in _rust_use_bindings(arg, source) if grouped else []: + if binding_name := _rust_use_module(binding): + add_edge(file_nid, _make_id(binding_name), "imports_from", line, context="import") + edges[-1]["_rust_use_binding"] = True # not call evidence: v8 had no such edge return for child in node.children: diff --git a/tests/test_rust_use_trees.py b/tests/test_rust_use_trees.py new file mode 100644 index 000000000..5dc040ab7 --- /dev/null +++ b/tests/test_rust_use_trees.py @@ -0,0 +1,209 @@ +"""A Rust `use` tree imports what the plain `use` of each of its bindings would. + +extract_rust cut a `use` at its first `{` and kept one segment, so +`use crate::util::{parse, render};` became one `imports_from` edge to a `util` +stub and `use {a::b, c};` none at all. Each binding of a tree now gets the edge +v8 emits for the equivalent plain `use` (`use crate::util::parse;` and +`use crate::util::render;`), from the same file node and on the binding's line. +Plain `use` declarations are unchanged, and so are call edges: v8's edge for a +group still serves the call pass, which does not see the new ones. +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +import graphify.extractors.rust as rust_module +from graphify.extract import extract + +_CARGO = '[package]\nname = "demo"\nversion = "0.1.0"\nedition = "2021"\n' +_FILES = {"src/lib.rs": "pub mod util;\npub mod user;\n", "src/util.rs": "pub fn parse() {}\npub fn render() {}\n"} + + +def _write(root: Path, files: dict[str, str]) -> None: + for name, body in {"Cargo.toml": _CARGO, **files}.items(): + (root / name).parent.mkdir(parents=True, exist_ok=True) + (root / name).write_text(body, encoding="utf-8") + + +def _extract(root: Path, files: dict[str, str]) -> dict: + _write(root, files) + paths = sorted(root / name for name in files if name.endswith(".rs")) + return extract(paths, root=root, cache_root=root / "graphify-out", parallel=False) + + +def _build(root: Path, changed: list[Path] | None = None) -> dict: + from graphify.watch import _rebuild_code + + assert _rebuild_code(root, changed_paths=changed, no_cluster=True, acquire_lock=False) is True + return json.loads((root / "graphify-out" / "graph.json").read_text(encoding="utf-8")) + + +def _edges(edges: list[dict], kind: str = "import") -> list[str]: + return sorted(json.dumps(e, sort_keys=True) for e in edges if e.get("context") == kind) + + +# name: (the tree, and the same bindings as plain `use` declarations on the +# same lines). Each pair must give the same edges and nodes. +SHAPES = { + "group": ("use crate::util::{parse, render};\n", "use crate::util::parse; use crate::util::render;\n"), + "nested": ("use crate::{util::{parse, render}, lib};\n", + "use crate::util::parse; use crate::util::render; use crate::lib;\n"), + "self": ("use crate::util::{self, parse};\n", "use crate::util; use crate::util::parse;\n"), + "self_alias": ("use crate::util::{self as u};\n", "use crate::util as u;\n"), + "alias": ("use crate::util::{parse as p, render};\n", "use crate::util::parse as p; use crate::util::render;\n"), + "alias_comment": ("use crate::util::{parse /* note */ as p};\n", "use crate::util::parse /* note */ as p;\n"), + "alias_after_as_comment": ("use crate::util::{parse as /* note */ p};\n", "use crate::util::parse as /* note */ p;\n"), + "self_alias_comment": ("use crate::util::{self /* note */ as u};\n", "use crate::util /* note */ as u;\n"), + "spaced_glob_delimiter": ("use crate::util :: {*};\n", "use crate::util :: *;\n"), + "self_delimiter_comment": ("use crate::util /* note */ :: {self};\n", "use crate::util;\n"), + "nested_self_prefix_comment": ("use crate::/* note */{util::{self}};\n", "use crate::/* note */util;\n"), + "nested_glob_prefix_comment": ("use crate::/* note */{util::{*}};\n", "use crate::/* note */util::*;\n"), + "nested_prefix_brace_comment": ("use crate::/* { */util::{nested::{parse}};\n", "use crate::/* { */util::nested::parse;\n"), + "glob": ("use crate::{util::*, lib};\n", "use crate::util::*; use crate::lib;\n"), + "glob_alone": ("use crate::util::{*};\n", "use crate::util::*;\n"), + "root_group": ("use {crate::util::parse, std::io};\n", "use crate::util::parse; use std::io;\n"), + "leading_colons": ("use ::std::{io, fmt::Write};\n", "use ::std::io; use ::std::fmt::Write;\n"), + "root_colons": ("use ::{std::io};\n", "use ::std::io;\n"), + "braces": ("use std::{{self}, {io::*}};\n", "use std; use std::io::*;\n"), + "super": ("use super::{util, lib::Thing};\n", "use super::util; use super::lib::Thing;\n"), + "pub": ("pub use self::{a::B, c};\n", "pub use self::a::B; pub use self::c;\n"), + "raw": ("use crate::{r#match, util::r#type};\n", "use crate::r#match; use crate::util::r#type;\n"), + "comments": ("use crate::util::{/* first */ parse, // second\nrender, a::/* in */b};\n", + "use crate::util::parse;\nuse crate::util::render; use crate::util::a::/* in */b;\n"), + "spacing": ("use crate :: util::{ parse as p , render };\n", "use crate :: util::parse as p; use crate :: util::render;\n"), + "lines": ("use crate::util::{\n parse,\n render as draw,\n};\n", + "\nuse crate::util::parse;\nuse crate::util::render as draw;\n\n"), +} + + +@pytest.mark.parametrize("name", sorted(SHAPES)) +def test_a_tree_imports_what_its_plain_list_does(tmp_path: Path, name: str): + tree, plain = SHAPES[name] + assert tree.count("\n") == plain.count("\n") + graphs = {} + for form, text in (("tree", tree), ("plain", plain)): + root = tmp_path / form + result = _extract(root, {**_FILES, "src/user.rs": text + "pub fn run() {}\n"}) + graphs[form] = (_edges(result["edges"]), sorted(json.dumps(n, sort_keys=True) for n in result["nodes"]), + _build(root)) + assert graphs["tree"][:2] == graphs["plain"][:2] + persisted = {form: (_edges(graph["links"]), sorted(json.dumps(n, sort_keys=True) for n in graph["nodes"])) + for form, (_e, _n, graph) in graphs.items()} + assert persisted["tree"] == persisted["plain"] + + +def test_each_binding_gets_the_plain_use_target(tmp_path: Path): + result = _extract(tmp_path, {**_FILES, "src/user.rs": ( + "use crate::util::{self, parse, render as draw, nested::{deep, *}};\nuse {a::b, c};\n" + "use crate::util::parse;\nuse std::io::Read;\nuse foo::bar as baz;\nuse crate::util::*;\npub use\n std::fmt;\n")}) + edges = sorted((e["relation"], e["target"], e["source_location"]) for e in result["edges"] + if e.get("context") == "import") + assert edges == sorted([ + ("imports_from", "util", "L1"), ("imports_from", "parse", "L1"), ("imports_from", "render_as_draw", "L1"), + ("imports_from", "deep", "L1"), ("imports_from", "nested", "L1"), ("imports_from", "b", "L2"), + ("imports_from", "c", "L2"), + # plain declarations, exactly as v8 emits them + ("imports_from", "parse", "L3"), ("imports_from", "read", "L4"), ("imports_from", "bar_as_baz", "L5"), + ("imports_from", "util", "L6"), ("imports_from", "fmt", "L7")]) + sources = {e["source"] for e in result["edges"] if e.get("context") == "import"} + file_node = next(n["id"] for n in result["nodes"] if n.get("label") == "user.rs") + assert sources == {file_node} + assert all(e["source_file"].endswith("user.rs") for e in result["edges"] if e.get("context") == "import") + assert not any(k.startswith("_rust_use") for e in result["edges"] for k in e) + + +def test_a_file_whose_only_use_is_a_root_group_gets_edges_from_its_file_node(tmp_path: Path): + # v8 emitted nothing for `use {..};`. + result = _extract(tmp_path, {**_FILES, "src/user.rs": "use {crate::util::parse, std::io};\n"}) + file_node = next(n["id"] for n in result["nodes"] if n.get("label") == "user.rs") + assert sorted((e["source"], e["target"]) for e in result["edges"] if e.get("context") == "import") == [ + (file_node, "io"), (file_node, "parse")] + + +# Files at the scan root, so file node ids are bare stems: the binding `util` +# names util.rs's file node, which v8's group edge (`crate`) never did. Were +# the binding call evidence, run() -> helper() would be promoted to EXTRACTED. +_CALLS = {"lib.rs": "pub mod util;\npub mod user;\npub mod other;\n", + "util.rs": "pub fn helper() {}\npub fn shared() {}\n", "other.rs": "pub fn shared() {}\n", + "user.rs": "use crate::{util, other::Thing};\npub fn run() { helper(); shared(); }\n"} + + +def test_calls_are_the_ones_v8_finds(tmp_path: Path, monkeypatch): + def calls(root: Path) -> list[str]: + result = extract(sorted(root / name for name in _CALLS), root=root, cache_root=root / "out", parallel=False) + return sorted(json.dumps({k: e.get(k) for k in ("source", "target", "relation", "confidence", + "confidence_score", "source_location")}, sort_keys=True) + for e in result["edges"] if e.get("context") in ("call", "constructor")) + + for name, body in _CALLS.items(): + for form in ("fix", "v8"): + (tmp_path / form / name).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / form / name).write_text(body, encoding="utf-8") + fixed = calls(tmp_path / "fix") + monkeypatch.setattr(rust_module, "_rust_use_bindings", lambda *args: []) # v8: no binding edges + assert fixed and fixed == calls(tmp_path / "v8") + + +def _parity(tmp_path: Path, monkeypatch, steps: list) -> None: + """Build, then apply each change: watch and `graphify extract` equal a full build.""" + import graphify.__main__ as mainmod + + def cli(root: Path) -> dict: + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr(mainmod.sys, "argv", ["graphify", "extract", str(root), "--code-only", "--no-cluster"]) + try: + mainmod.main() + except SystemExit as exc: + assert exc.code in (None, 0) + return json.loads((root / "graphify-out" / "graph.json").read_text(encoding="utf-8")) + + def view(graph: dict) -> tuple: + # v8 keeps external stubs across incremental builds (unreferenced ones too) + # and stamps them `semantic` on reload; compare the rest exactly. + return (sorted(json.dumps(e, sort_keys=True) for e in graph.get("links", graph.get("edges", []))), + sorted(json.dumps(n, sort_keys=True) for n in graph["nodes"] if not n.get("external"))) + + files = {**_FILES, "src/user.rs": "use crate::util::{parse, render};\n", "src/other.rs": "use crate::util::parse;\n"} + for mode in ("watch", "extract"): + root = tmp_path / mode + _write(root, files) + _build(root) if mode == "watch" else cli(root) + for number, (name, text) in enumerate(steps): + (root / name).write_text(text, encoding="utf-8") + incremental = _build(root, [root / name]) if mode == "watch" else cli(root) + fresh = tmp_path / f"{mode}-fresh-{number}" + _write(fresh, {path: (root / path).read_text(encoding="utf-8") for path in files}) + assert view(incremental) == view(_build(fresh) if mode == "watch" else cli(fresh)), (mode, name) + + +def test_incremental_builds_match_full_builds(tmp_path: Path, monkeypatch): + _parity(tmp_path, monkeypatch, [ + ("src/user.rs", "use crate::util::{parse, render as draw, self};\n"), # grouped edited + ("src/other.rs", "use {crate::util::render, std::io};\n"), # plain becomes a root group + ("src/user.rs", "use crate::util::parse;\n"), # group becomes plain + ]) + + +def test_a_cache_entry_from_before_the_bindings_is_not_served(tmp_path: Path, monkeypatch): + import graphify.cache as cache_mod + from graphify.cache import save_cached + + files = {**_FILES, "src/user.rs": "use crate::util::{parse, render};\n"} + _write(tmp_path, files) + path = tmp_path / "src" / "user.rs" + current = cache_mod._AST_CACHE_SCHEMA + monkeypatch.setattr(cache_mod, "_AST_CACHE_SCHEMA", 6) # v0.9.80's + with monkeypatch.context() as patch: + patch.setattr(rust_module, "_rust_use_bindings", lambda *args: []) # what v8 cached + stale = rust_module.extract_rust(path) + for edge in stale["edges"]: + edge.pop("_rust_use_group", None) + save_cached(path, stale, tmp_path, cache_root=tmp_path / "graphify-out") + monkeypatch.setattr(cache_mod, "_AST_CACHE_SCHEMA", current) + result = extract(sorted(tmp_path / name for name in files), root=tmp_path, cache_root=tmp_path / "graphify-out", + parallel=False) + assert sorted(e["target"] for e in result["edges"] + if e.get("context") == "import" and e["source_file"].endswith("user.rs")) == ["parse", "render"]