From b56a415388ec20d76453366e568946dfd2cd99bd Mon Sep 17 00:00:00 2001 From: Giovane Cesar da Silva Date: Thu, 11 Jun 2026 15:08:36 -0600 Subject: [PATCH] feat(extract): add opt-in --cargo crate dependency extractor Closes #1264 Also fixes a narrow watch-mode path-filtering bug found while running the required local full-suite gate for this PR. Hidden directories inside the watched corpus are still ignored; hidden ancestors outside the watched root no longer suppress valid file events. --- README.md | 1 + graphify/__main__.py | 25 ++- graphify/cargo_introspect.py | 98 +++++++++ graphify/watch.py | 10 +- tests/test_cargo_introspect.py | 365 +++++++++++++++++++++++++++++++++ tests/test_watch.py | 21 +- 6 files changed, 506 insertions(+), 14 deletions(-) create mode 100644 graphify/cargo_introspect.py create mode 100644 tests/test_cargo_introspect.py diff --git a/README.md b/README.md index d4790ad7..65954025 100644 --- a/README.md +++ b/README.md @@ -574,6 +574,7 @@ graphify extract ./docs --backend claude-cli # route through Claude Code CLI - graphify extract ./docs --backend azure # Azure OpenAI (set AZURE_OPENAI_API_KEY + AZURE_OPENAI_ENDPOINT) graphify extract ./docs --max-workers 16 # AST parallelism (also GRAPHIFY_MAX_WORKERS) graphify extract --postgres "postgresql://user:pass@host/db" # introspect live PostgreSQL schema directly +graphify extract ./my-workspace --cargo # introspect Rust Cargo workspace dependencies directly graphify extract ./docs --token-budget 30000 # smaller semantic chunks for local/small models graphify extract ./docs --max-concurrency 2 # fewer parallel LLM calls (useful for local inference) graphify extract ./docs --api-timeout 900 # longer HTTP timeout for slow local models (default 600s) diff --git a/graphify/__main__.py b/graphify/__main__.py index 192af325..d700318f 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -2182,6 +2182,7 @@ def main() -> None: print(" --postgres DSN extract schema from a live PostgreSQL database") print(" maps tables, views, functions + FK relationships;") print(" column-level detail is not represented in the graph") + print(" --cargo extract crate→crate deps from Cargo.toml") print(" --global also merge the resulting graph into the global graph") print(" --as repo tag for --global (default: target directory name)") print(" global add add/update a project graph in the global graph (~/.graphify/global-graph.json)") @@ -3904,7 +3905,7 @@ def main() -> None: "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " "[--model M] [--mode deep] [--out DIR] [--google-workspace] [--no-cluster] " "[--max-workers N] [--token-budget N] [--max-concurrency N] " - "[--api-timeout S] [--postgres DSN]", + "[--api-timeout S] [--postgres DSN] [--cargo]", file=sys.stderr, ) sys.exit(1) @@ -3924,6 +3925,7 @@ def main() -> None: extract_mode: str | None = None out_dir: Path | None = None cli_postgres_dsn: str | None = None + cli_cargo: bool = False no_cluster = False dedup_llm = False google_workspace = False @@ -4023,6 +4025,9 @@ def main() -> None: cli_postgres_dsn = args[i + 1]; i += 2 elif a.startswith("--postgres="): cli_postgres_dsn = a.split("=", 1)[1]; i += 1 + elif a == "--cargo": + cli_cargo = True + i += 1 else: i += 1 @@ -4322,13 +4327,25 @@ def main() -> None: print(f"[graphify extract] PostgreSQL: {len(pg_result['nodes'])} nodes, " f"{len(pg_result['edges'])} edges") - # Merge AST + semantic + pg_result. Order matters for deduplication: passing AST + cargo_result: dict = {"nodes": [], "edges": []} + if cli_cargo: + from graphify.cargo_introspect import introspect_cargo + print("[graphify extract] introspecting Cargo workspace...") + try: + cargo_result = introspect_cargo(target) + except (ConnectionError, ImportError) as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + print(f"[graphify extract] Cargo: {len(cargo_result['nodes'])} nodes, " + f"{len(cargo_result['edges'])} edges") + + # Merge AST + semantic + pg_result + cargo_result. Order matters for deduplication: passing AST # first means semantic node attributes win on collision (richer labels # for symbols also referenced in docs). Hyperedges only come from the # semantic side. merged: dict = { - "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])), - "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])), + "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])) + list(cargo_result.get("nodes", [])), + "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])) + list(cargo_result.get("edges", [])), "hyperedges": list(sem_result.get("hyperedges", [])), "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0), "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0), diff --git a/graphify/cargo_introspect.py b/graphify/cargo_introspect.py new file mode 100644 index 00000000..3bda9612 --- /dev/null +++ b/graphify/cargo_introspect.py @@ -0,0 +1,98 @@ +"""Cargo manifest introspection for workspace-internal crate dependencies.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + + +_CONFIDENCE_EXTRACTED = "EXTRACTED" + + +def _load_toml(path: Path) -> dict[str, Any]: + try: + import tomllib # type: ignore[import-not-found] + except ModuleNotFoundError: + try: + import tomli as tomllib # type: ignore[import-not-found,no-redef] + except ModuleNotFoundError: + raise ImportError( + "--cargo on Python 3.10 needs tomli. Install with: pip install tomli" + ) from None + + with path.open("rb") as manifest: + return tomllib.load(manifest) + + +def _member_manifest_paths(root: Path, root_data: dict[str, Any]) -> list[Path]: + paths: list[Path] = [] + if isinstance(root_data.get("package"), dict): + paths.append(root / "Cargo.toml") + + workspace = root_data.get("workspace") + members = workspace.get("members", []) if isinstance(workspace, dict) else [] + if not isinstance(members, list): + return paths + + for pattern in members: + if not isinstance(pattern, str): + continue + for member in sorted(root.glob(pattern)): + manifest = member / "Cargo.toml" + if manifest.is_file() and manifest not in paths: + paths.append(manifest) + return paths + + +def introspect_cargo(root: str | Path) -> dict[str, Any]: + """Return crate nodes and internal dependency edges from Cargo manifests.""" + root_path = Path(root).resolve() + root_manifest = root_path / "Cargo.toml" + root_data = _load_toml(root_manifest) + + manifests = _member_manifest_paths(root_path, root_data) + crates: dict[str, tuple[str, Path, dict[str, Any]]] = {} + + for manifest in manifests: + data = root_data if manifest == root_manifest else _load_toml(manifest) + package = data.get("package") + if not isinstance(package, dict): + continue + name = package.get("name") + if isinstance(name, str): + crates[name] = (f"crate:{name}", manifest, data) + + nodes = [ + { + "id": crate_id, + "label": name, + "source_file": manifest.relative_to(root_path).as_posix(), + "source_location": "L1", + } + for name, (crate_id, manifest, _data) in sorted(crates.items()) + ] + + edges: list[dict[str, Any]] = [] + for source_name, (source_id, manifest, data) in sorted(crates.items()): + dependencies = data.get("dependencies", {}) + if not isinstance(dependencies, dict): + continue + source_file = manifest.relative_to(root_path).as_posix() + for dependency_name in sorted(dependencies): + target = crates.get(dependency_name) + if target is None: + continue + edges.append( + { + "source": source_id, + "target": target[0], + "relation": "crate_depends_on", + "context": "cargo_dependency", + "weight": 1.0, + "confidence": _CONFIDENCE_EXTRACTED, + "source_file": source_file, + "source_location": "L1", + } + ) + + return {"nodes": nodes, "edges": edges} diff --git a/graphify/watch.py b/graphify/watch.py index 712f9fc6..b6eec512 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -838,7 +838,7 @@ def watch(watch_path: Path, debounce: float = 3.0) -> None: nonlocal last_trigger, pending if event.is_directory: return - path = Path(event.src_path) + path = Path(os.fsdecode(event.src_path)) # Check .graphifyignore BEFORE the extension/dotfile/out filters so # the cheapest short-circuit for users with broad ignore patterns # (node_modules/, .venv/, build/, …) fires first. _is_ignored @@ -848,9 +848,13 @@ def watch(watch_path: Path, debounce: float = 3.0) -> None: return if path.suffix.lower() not in _WATCHED_EXTENSIONS: return - if any(part.startswith(".") for part in path.parts): + try: + filter_parts = path.relative_to(watch_root_for_ignore).parts + except ValueError: + filter_parts = path.parts + if any(part.startswith(".") for part in filter_parts): return - if _GRAPHIFY_OUT in path.parts: + if _GRAPHIFY_OUT in filter_parts: return last_trigger = time.monotonic() pending = True diff --git a/tests/test_cargo_introspect.py b/tests/test_cargo_introspect.py new file mode 100644 index 00000000..e80d8c16 --- /dev/null +++ b/tests/test_cargo_introspect.py @@ -0,0 +1,365 @@ +import pytest + +from graphify.cargo_introspect import introspect_cargo + + +def _write_manifest(path, content): + path.write_text(content.lstrip(), encoding="utf-8") + + +def test_cargo_introspect_workspace_internal_dependency_only(tmp_path): + """Real workspace: pin raw graph fields while excluding registry-only deps.""" + # This exercises actual Cargo.toml discovery from disk, proving internal path + # dependencies become edges while external registry packages stay out of the graph. + _write_manifest( + tmp_path / "Cargo.toml", + """ +[workspace] +members = ["app", "core"] +""", + ) + app = tmp_path / "app" + core = tmp_path / "core" + app.mkdir() + core.mkdir() + _write_manifest( + app / "Cargo.toml", + """ +[package] +name = "app" +version = "0.1.0" +edition = "2021" + +[dependencies] +core = { path = "../core" } +serde = "1" +""", + ) + _write_manifest( + core / "Cargo.toml", + """ +[package] +name = "core" +version = "0.1.0" +edition = "2021" +""", + ) + + result = introspect_cargo(tmp_path) + + node_ids = {node["id"] for node in result["nodes"]} + assert node_ids == {"crate:app", "crate:core"} + assert "crate:serde" not in node_ids + assert { + "id": "crate:app", + "label": "app", + "source_file": "app/Cargo.toml", + "source_location": "L1", + } in result["nodes"] + + assert { + "source": "crate:app", + "target": "crate:core", + "relation": "crate_depends_on", + "context": "cargo_dependency", + "weight": 1.0, + "confidence": "EXTRACTED", + "source_file": "app/Cargo.toml", + "source_location": "L1", + } in result["edges"] + assert not any( + edge["source"] == "crate:app" and edge["target"] == "crate:serde" + for edge in result["edges"] + ) + + +def test_cargo_introspect_malformed_toml_reports_parser_error(tmp_path): + """Malformed manifests surface the TOML parser failure, not an arbitrary crash.""" + # Pin the class name so this works with stdlib tomllib and Python 3.10 tomli. + _write_manifest( + tmp_path / "Cargo.toml", + """ +[package +name = "broken" +""", + ) + + with pytest.raises(Exception) as exc_info: + introspect_cargo(tmp_path) + + assert exc_info.type.__name__ == "TOMLDecodeError" + + +def test_cargo_introspect_degenerate_manifests_return_empty_or_skip_bad_deps(tmp_path): + """Degenerate but parseable manifests should not invent graph data or crash.""" + # Empty and nameless packages prove crate nodes require package identity; the + # scalar dependencies case proves malformed dependency sections are ignored safely. + empty_manifest = tmp_path / "empty" + empty_manifest.mkdir() + _write_manifest(empty_manifest / "Cargo.toml", "") + + empty_result = introspect_cargo(empty_manifest) + + assert empty_result["nodes"] == [] + assert empty_result["edges"] == [] + + nameless_package = tmp_path / "nameless" + nameless_package.mkdir() + _write_manifest( + nameless_package / "Cargo.toml", + """ +[package] +version = "0.1.0" +""", + ) + + nameless_result = introspect_cargo(nameless_package) + + assert nameless_result["nodes"] == [] + assert nameless_result["edges"] == [] + + scalar_dependencies = tmp_path / "scalar-dependencies" + scalar_dependencies.mkdir() + _write_manifest( + scalar_dependencies / "Cargo.toml", + """ +[package] +name = "app" +version = "0.1.0" + +dependencies = "not-a-table" +""", + ) + + scalar_result = introspect_cargo(scalar_dependencies) + + assert scalar_result["nodes"] == [ + { + "id": "crate:app", + "label": "app", + "source_file": "Cargo.toml", + "source_location": "L1", + } + ] + assert scalar_result["edges"] == [] + + +def test_cargo_introspect_old_manifest_keeps_internal_path_dep_and_skips_external(tmp_path): + """Legacy manifests still resolve path deps and ignore bare-string externals.""" + # Older Cargo files may omit modern metadata and use bare version strings; the + # graph should keep only workspace-internal relationships. + _write_manifest( + tmp_path / "Cargo.toml", + """ +[workspace] +members = ["legacy", "internal"] +""", + ) + legacy = tmp_path / "legacy" + internal = tmp_path / "internal" + legacy.mkdir() + internal.mkdir() + _write_manifest( + legacy / "Cargo.toml", + """ +[package] +name = "legacy" +version = "0.1.0" + +[dependencies] +rand = "0.8" +internal = { path = "../internal" } +""", + ) + _write_manifest( + internal / "Cargo.toml", + """ +[package] +name = "internal" +version = "0.1.0" +""", + ) + + result = introspect_cargo(tmp_path) + + node_ids = {node["id"] for node in result["nodes"]} + edge_pairs = {(edge["source"], edge["target"]) for edge in result["edges"]} + assert node_ids == {"crate:legacy", "crate:internal"} + assert "crate:rand" not in node_ids + assert len(result["edges"]) == 1 + assert ("crate:legacy", "crate:internal") in edge_pairs + assert ("crate:legacy", "crate:rand") not in edge_pairs + + +def test_cargo_introspect_modern_virtual_and_root_package_workspaces(tmp_path): + """Modern workspace forms cover virtual roots, workspace deps, and root packages.""" + # Virtual manifests and root-package workspaces discover members differently; + # both must produce exact internal graph shapes without registry-only edges. + virtual_root = tmp_path / "virtual" + virtual_root.mkdir() + _write_manifest( + virtual_root / "Cargo.toml", + """ +[workspace] +members = ["crates/*"] + +[workspace.dependencies] +beta = { path = "crates/beta" } +serde = "1" +""", + ) + alpha = virtual_root / "crates" / "alpha" + beta = virtual_root / "crates" / "beta" + alpha.mkdir(parents=True) + beta.mkdir(parents=True) + _write_manifest( + alpha / "Cargo.toml", + """ +[package] +name = "alpha" +version = "0.1.0" +edition = "2021" + +[dependencies] +beta = { workspace = true } +serde = { workspace = true } +""", + ) + _write_manifest( + beta / "Cargo.toml", + """ +[package] +name = "beta" +version = "0.1.0" +edition = "2021" +""", + ) + + virtual_result = introspect_cargo(virtual_root) + + assert {node["id"] for node in virtual_result["nodes"]} == { + "crate:alpha", + "crate:beta", + } + assert len(virtual_result["nodes"]) == 2 + assert len(virtual_result["edges"]) == 1 + assert { + "source": "crate:alpha", + "target": "crate:beta", + "relation": "crate_depends_on", + "context": "cargo_dependency", + "weight": 1.0, + "confidence": "EXTRACTED", + "source_file": "crates/alpha/Cargo.toml", + "source_location": "L1", + } in virtual_result["edges"] + + package_root = tmp_path / "package-root" + package_root.mkdir() + _write_manifest( + package_root / "Cargo.toml", + """ +[package] +name = "root_pkg" +version = "0.1.0" +edition = "2021" + +[workspace] +members = ["crates/*"] +""", + ) + member = package_root / "crates" / "member" + member.mkdir(parents=True) + _write_manifest( + member / "Cargo.toml", + """ +[package] +name = "member" +version = "0.1.0" +edition = "2021" + +[dependencies] +root_pkg = { path = "../.." } +""", + ) + + package_result = introspect_cargo(package_root) + + assert {node["id"] for node in package_result["nodes"]} == { + "crate:root_pkg", + "crate:member", + } + assert len(package_result["nodes"]) == 2 + assert len(package_result["edges"]) == 1 + assert { + "source": "crate:member", + "target": "crate:root_pkg", + "relation": "crate_depends_on", + "context": "cargo_dependency", + "weight": 1.0, + "confidence": "EXTRACTED", + "source_file": "crates/member/Cargo.toml", + "source_location": "L1", + } in package_result["edges"] + + +def test_cargo_introspect_large_workspace_dependency_chain(tmp_path): + """Large deterministic workspace proves chain extraction scales by shape, not timing.""" + # The exact 200-node/199-edge chain guards against truncation, glob misses, or + # accidental timing-based assertions that would make the test flaky. + crate_count = 200 + _write_manifest( + tmp_path / "Cargo.toml", + """ +[workspace] +members = ["crates/*"] +""", + ) + + for index in range(crate_count): + crate_dir = tmp_path / "crates" / f"crate_{index:03d}" + crate_dir.mkdir(parents=True) + dependency_block = "" + if index + 1 < crate_count: + dependency_block = f""" + +[dependencies] +crate_{index + 1:03d} = {{ path = "../crate_{index + 1:03d}" }} +""" + _write_manifest( + crate_dir / "Cargo.toml", + f''' +[package] +name = "crate_{index:03d}" +version = "0.1.0" +edition = "2021"{dependency_block} +''', + ) + + result = introspect_cargo(tmp_path) + + assert len(result["nodes"]) == crate_count + assert len(result["edges"]) == crate_count - 1 + assert {node["id"] for node in result["nodes"]} == { + f"crate:crate_{index:03d}" for index in range(crate_count) + } + assert { + "source": "crate:crate_000", + "target": "crate:crate_001", + "relation": "crate_depends_on", + "context": "cargo_dependency", + "weight": 1.0, + "confidence": "EXTRACTED", + "source_file": "crates/crate_000/Cargo.toml", + "source_location": "L1", + } in result["edges"] + assert { + "source": "crate:crate_198", + "target": "crate:crate_199", + "relation": "crate_depends_on", + "context": "cargo_dependency", + "weight": 1.0, + "confidence": "EXTRACTED", + "source_file": "crates/crate_198/Cargo.toml", + "source_location": "L1", + } in result["edges"] diff --git a/tests/test_watch.py b/tests/test_watch.py index 7d984a6f..8dcdf499 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -424,9 +424,11 @@ def test_watch_handler_honors_graphifyignore(tmp_path, monkeypatch): import threading from graphify import watch as watch_mod - (tmp_path / ".graphifyignore").write_text("node_modules/\nbuild/\n", encoding="utf-8") - (tmp_path / "node_modules").mkdir() - (tmp_path / "build").mkdir() + watch_root = tmp_path / ".hidden-parent" / "corpus" + watch_root.mkdir(parents=True) + (watch_root / ".graphifyignore").write_text("node_modules/\nbuild/\n", encoding="utf-8") + (watch_root / "node_modules").mkdir() + (watch_root / "build").mkdir() rebuild_calls: list[Path] = [] notify_calls: list[Path] = [] @@ -435,19 +437,24 @@ def test_watch_handler_honors_graphifyignore(tmp_path, monkeypatch): # Run watch() in a thread with a short debounce so we can verify the # post-debounce dispatch path actually runs on real events. - t = threading.Thread(target=watch_mod.watch, args=(tmp_path,), kwargs={"debounce": 0.2}, daemon=True) + t = threading.Thread( + target=watch_mod.watch, + args=(watch_root,), + kwargs={"debounce": 0.2}, + daemon=True, + ) t.start() time.sleep(0.5) # let observer.start() settle # Ignored writes — handler must drop these. - (tmp_path / "node_modules" / "junk.js").write_text("// noise\n", encoding="utf-8") - (tmp_path / "build" / "out.py").write_text("x = 1\n", encoding="utf-8") + (watch_root / "node_modules" / "junk.js").write_text("// noise\n", encoding="utf-8") + (watch_root / "build" / "out.py").write_text("x = 1\n", encoding="utf-8") time.sleep(1.0) assert rebuild_calls == [], "ignored writes triggered a rebuild" assert notify_calls == [], "ignored writes triggered a notify" # Non-ignored write — handler must accept and (after debounce) dispatch. - (tmp_path / "app.py").write_text("def f():\n return 1\n", encoding="utf-8") + (watch_root / "app.py").write_text("def f():\n return 1\n", encoding="utf-8") deadline = time.monotonic() + 5.0 while time.monotonic() < deadline and not rebuild_calls: time.sleep(0.1)