diff --git a/graphify/__main__.py b/graphify/__main__.py index ffb7db90..708c4581 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -117,6 +117,18 @@ from graphify.install import ( # noqa: E402,F401 _OPENCODE_CONFIG_PATH, _PLATFORM_CONFIG, ) +from graphify.cli import ( # noqa: E402,F401 + dispatch_command, + _StageTimer, + _clone_repo, + _default_graph_path, + _enforce_graph_size_cap_or_exit, + _run_hook_guard, + _SEARCH_NUDGE, + _READ_NUDGE, + _HOOK_SOURCE_EXTS, + _GEMINI_NUDGE_TEXT, +) @@ -142,53 +154,10 @@ def __getattr__(name: str) -> str: raise AttributeError(f"module {__name__!r} has no attribute {name!r}") -def _default_graph_path() -> str: - return str(Path(_GRAPHIFY_OUT) / "graph.json") -class _StageTimer: - """Print per-stage wall-clock timings to stderr when --timing is set (#1490). - - Monotonic (perf_counter), diagnostic-only: emits ``[graphify timing] : - N.Ns`` after each stage and a final total. Off by default, so normal output is - byte-identical and machine-read stdout is untouched. - """ - - def __init__(self, enabled: bool) -> None: - import time as _time - self._now = _time.perf_counter - self.enabled = enabled - self.start = self._now() - self._last = self.start - - def mark(self, stage: str) -> None: - now = self._now() - if self.enabled: - print(f"[graphify timing] {stage}: {now - self._last:.1f}s", file=sys.stderr) - self._last = now - - def total(self) -> None: - if self.enabled: - print(f"[graphify timing] total: {self._now() - self.start:.1f}s", file=sys.stderr) -def _enforce_graph_size_cap_or_exit(gp: Path) -> None: - """Reject oversized graph files before parsing (CLI exit-on-fail flavor). - - Delegates to ``graphify.security.check_graph_file_size_cap`` and turns the - raised ``ValueError`` into a CLI-style ``error: ...`` message + exit 1. - Use this from ``__main__.py`` subcommands that already use the ``print + - sys.exit(1)`` idiom. Library/MCP/loader callers (``serve._load_graph``, - ``build``, ``benchmark``, ``tree_html``, ``callflow_html``, ``prs``, - ``global_graph``, ``watch``, ``export``) call the security helper directly - and let the ``ValueError`` propagate. - """ - from graphify.security import check_graph_file_size_cap - try: - check_graph_file_size_cap(gp) - except ValueError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) def _check_skill_version(skill_dst: Path) -> None: @@ -284,40 +253,11 @@ def _version_tuple(version: str) -> tuple[int, ...]: # additionalContext on PreToolUse (Codex Desktop does not — that path stays a # no-op via `hook-check`). Compact separators keep the payload byte-for-byte the # same JSON the old `echo` emitted. -_SEARCH_NUDGE = json.dumps({ - "hookSpecificOutput": { - "hookEventName": "PreToolUse", - "additionalContext": ( - 'MANDATORY: graphify-out/graph.json exists. You MUST run ' - '`graphify query ""` before grepping raw files. Only grep ' - 'after graphify has oriented you, or to modify/debug specific lines.' - ), - } -}, ensure_ascii=False, separators=(",", ":")) + "\n" -_READ_NUDGE = json.dumps({ - "hookSpecificOutput": { - "hookEventName": "PreToolUse", - "additionalContext": ( - 'MANDATORY: graphify-out/graph.json exists. You MUST run graphify ' - 'before reading source files. Use: `graphify query ""` ' - '(scoped subgraph), `graphify explain ""`, or ' - '`graphify path "" ""`. Only read raw files after graphify has ' - 'oriented you, or to modify/debug specific lines. This rule applies to ' - 'subagents too — include it in every subagent prompt involving code ' - 'exploration.' - ), - } -}, ensure_ascii=False, separators=(",", ":")) + "\n" # Source/doc extensions the Read|Glob guard nudges on (verbatim from the old hook). # The trailing-extension test (real final path segment, then its last '.') means # '.json' never false-matches '.js', and framework files like '.astro' are kept. -_HOOK_SOURCE_EXTS = ( - '.py', '.js', '.ts', '.tsx', '.jsx', '.astro', '.vue', '.svelte', '.go', - '.rs', '.java', '.rb', '.c', '.h', '.cpp', '.hpp', '.cc', '.cs', '.kt', - '.swift', '.php', '.scala', '.lua', '.sh', '.md', '.rst', '.txt', '.mdx', -) @@ -356,12 +296,6 @@ _HOOK_SOURCE_EXTS = ( # `python`/`py` or absent on Windows) and embedded backticks + escaped quotes that # Windows PowerShell mangles (#522 follow-up); the subcommand form has no such # dependency and parses under every shell. -_GEMINI_NUDGE_TEXT = ( - 'graphify: knowledge graph at graphify-out/. For focused questions, run ' - '`graphify query ""` (scoped subgraph, usually much smaller than ' - 'GRAPH_REPORT.md) instead of grepping raw files. Read GRAPH_REPORT.md only ' - 'for broad architecture context.' -) @@ -455,61 +389,6 @@ _CODEX_HOOK = { -def _run_hook_guard(kind: str) -> None: - """Shell-agnostic PreToolUse guard (#522). - - Reads the tool-call JSON from stdin and, when a knowledge graph exists in the - current output dir, prints a nudge (`additionalContext`) telling the agent to - use graphify instead of grepping/reading raw files. Replaces the old inline - bash hooks that failed to parse on Windows. Always fails open: any error, or a - non-matching tool call, prints nothing and the caller exits 0, so a legitimate - tool call is never blocked. Detection mirrors the previous hooks exactly. - """ - from graphify.paths import out_path, GRAPHIFY_OUT_NAME - # Gemini's BeforeTool hook takes no stdin and must ALWAYS return a decision so - # the tool is never blocked; the graph nudge is appended only when a graph - # exists. Handled before the stdin read below (which the search/read guards need). - if kind == "gemini": - payload = {"decision": "allow"} - try: - if out_path("graph.json").is_file(): - payload["additionalContext"] = _GEMINI_NUDGE_TEXT - except Exception: - pass - sys.stdout.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":"))) - return - try: - d = json.loads(sys.stdin.buffer.read().decode("utf-8", "replace")) - except Exception: - return - if not isinstance(d, dict): - return - t = d.get("tool_input", d) - if not isinstance(t, dict): - return - try: - if kind == "search": - cmd_str = str(t.get("command", "") or "") - # Same set the old `case` matched: *grep*, *ripgrep*, and rg/find/fd/ - # ack/ag as a token (name followed by a space). - if any(tok in cmd_str for tok in ("grep", "ripgrep", "rg ", "find ", "fd ", "ack ", "ag ")) \ - and out_path("graph.json").is_file(): - sys.stdout.write(_SEARCH_NUDGE) - elif kind == "read": - vals = [str(t.get("file_path") or ""), str(t.get("pattern") or ""), str(t.get("path") or "")] - j = " ".join(vals).lower().replace("\\", "/") - tails = [ - "." + seg.rsplit(".", 1)[-1] - for v in vals if v - for seg in [v.lower().replace("\\", "/").rsplit("/", 1)[-1]] - if "." in seg - ] - under_out = "graphify-out/" in j or (GRAPHIFY_OUT_NAME.lower() + "/") in j - if not under_out and any(tl in _HOOK_SOURCE_EXTS for tl in tails) \ - and out_path("graph.json").is_file(): - sys.stdout.write(_READ_NUDGE) - except Exception: - pass @@ -563,63 +442,6 @@ def _run_hook_guard(kind: str) -> None: -def _clone_repo( - url: str, branch: str | None = None, out_dir: Path | None = None -) -> Path: - """Clone a GitHub repo to a local cache dir and return the path. - - Clones into ~/.graphify/repos// by default so repeated - runs on the same URL reuse the existing clone (git pull instead of clone). - """ - import subprocess as _sp - import re as _re - - # Normalise URL — strip trailing .git if present - url = url.rstrip("/") - if not url.endswith(".git"): - git_url = url + ".git" - else: - git_url = url - url = url[:-4] - - # Extract owner/repo from URL - m = _re.search(r"github\.com[:/]([^/]+)/([^/]+?)(?:\.git)?$", url) - if not m: - print(f"error: not a recognised GitHub URL: {url}", file=sys.stderr) - sys.exit(1) - owner, repo = m.group(1), m.group(2) - - if out_dir: - dest = out_dir - else: - dest = Path.home() / ".graphify" / "repos" / owner / repo - - if branch and branch.startswith("-"): - print(f"error: invalid branch name: {branch!r}", file=sys.stderr) - sys.exit(1) - - if dest.exists(): - print(f"Repo already cloned at {dest} - pulling latest...", flush=True) - cmd = ["git", "-C", str(dest), "pull"] - if branch: - cmd += ["origin", "--", branch] - result = _sp.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - print(f"warning: git pull failed:\n{result.stderr}", file=sys.stderr) - else: - dest.parent.mkdir(parents=True, exist_ok=True) - print(f"Cloning {url} -> {dest} ...", flush=True) - cmd = ["git", "clone", "--depth", "1"] - if branch: - cmd += ["--branch", branch] - cmd += ["--", git_url, str(dest)] - result = _sp.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - print(f"error: git clone failed:\n{result.stderr}", file=sys.stderr) - sys.exit(1) - - print(f"Ready at: {dest}", flush=True) - return dest def main() -> None: @@ -844,2544 +666,7 @@ def main() -> None: if dispatch_install_cli(cmd): return - if cmd == "provider": - from graphify.llm import _custom_providers_path, BACKENDS - import json as _json - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - global_path = _custom_providers_path(global_=True) - - if subcmd == "list": - global_path.parent.mkdir(parents=True, exist_ok=True) - existing: dict = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - if not existing: - print("No custom providers registered.") - else: - for name in existing: - print(f" {name} ({existing[name].get('base_url', '')})") - - elif subcmd == "show": - name = sys.argv[3] if len(sys.argv) > 3 else "" - if not name: - print("Usage: graphify provider show ", file=sys.stderr) - sys.exit(1) - existing = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - if name not in existing: - print(f"Provider '{name}' not found.", file=sys.stderr) - sys.exit(1) - print(_json.dumps({name: existing[name]}, indent=2)) - - elif subcmd == "add": - args = sys.argv[3:] - name = args[0] if args and not args[0].startswith("-") else "" - if not name: - print("Usage: graphify provider add --base-url URL --default-model MODEL --env-key KEY", file=sys.stderr) - sys.exit(1) - if name in BACKENDS: - print(f"Error: '{name}' is a built-in provider and cannot be overridden.", file=sys.stderr) - sys.exit(1) - base_url = "" - default_model = "" - env_key = "" - pricing_input = 0.0 - pricing_output = 0.0 - i = 1 - while i < len(args): - a = args[i] - if a == "--base-url" and i + 1 < len(args): - base_url = args[i + 1]; i += 2 - elif a.startswith("--base-url="): - base_url = a.split("=", 1)[1]; i += 1 - elif a == "--default-model" and i + 1 < len(args): - default_model = args[i + 1]; i += 2 - elif a.startswith("--default-model="): - default_model = a.split("=", 1)[1]; i += 1 - elif a == "--env-key" and i + 1 < len(args): - env_key = args[i + 1]; i += 2 - elif a.startswith("--env-key="): - env_key = a.split("=", 1)[1]; i += 1 - elif a == "--pricing-input" and i + 1 < len(args): - pricing_input = float(args[i + 1]); i += 2 - elif a == "--pricing-output" and i + 1 < len(args): - pricing_output = float(args[i + 1]); i += 2 - else: - i += 1 - if not base_url or not default_model or not env_key: - print("Error: --base-url, --default-model, and --env-key are required.", file=sys.stderr) - sys.exit(1) - from graphify.llm import provider_base_url_ok - if not provider_base_url_ok(base_url, name): - print(f"Error: refusing to add provider with unsafe base_url {base_url!r}.", file=sys.stderr) - sys.exit(1) - global_path.parent.mkdir(parents=True, exist_ok=True) - existing = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - existing[name] = { - "base_url": base_url, - "default_model": default_model, - "env_key": env_key, - "pricing": {"input": pricing_input, "output": pricing_output}, - "temperature": 0, - } - global_path.write_text(_json.dumps(existing, indent=2) + "\n", encoding="utf-8") - print(f"Provider '{name}' added. Use with: graphify extract . --backend {name}") - - elif subcmd == "remove": - name = sys.argv[3] if len(sys.argv) > 3 else "" - if not name: - print("Usage: graphify provider remove ", file=sys.stderr) - sys.exit(1) - existing = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - if name not in existing: - print(f"Provider '{name}' not found.", file=sys.stderr) - sys.exit(1) - del existing[name] - global_path.write_text(_json.dumps(existing, indent=2) + "\n", encoding="utf-8") - print(f"Provider '{name}' removed.") - - else: - print("Usage: graphify provider [add|list|show|remove]", file=sys.stderr) - if subcmd: - sys.exit(1) - elif cmd == "prs": - from graphify.prs import cmd_prs - cmd_prs(sys.argv[2:]) - elif cmd == "hook": - from graphify.hooks import ( - install as hook_install, - uninstall as hook_uninstall, - status as hook_status, - ) - - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - print(hook_install(Path("."))) - elif subcmd == "uninstall": - print(hook_uninstall(Path("."))) - elif subcmd == "status": - print(hook_status(Path("."))) - else: - print("Usage: graphify hook [install|uninstall|status]", file=sys.stderr) - sys.exit(1) - elif cmd == "query": - if len(sys.argv) < 3: - print("Usage: graphify query \"\" [--dfs] [--context C] [--budget N] [--graph path]", file=sys.stderr) - sys.exit(1) - from graphify.serve import _query_graph_text - from graphify.security import sanitize_label - from networkx.readwrite import json_graph - from graphify import querylog - - question = sys.argv[2] - use_dfs = "--dfs" in sys.argv - budget = 2000 - graph_path = _default_graph_path() - context_filters: list[str] = [] - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--budget" and i + 1 < len(args): - try: - budget = int(args[i + 1]) - except ValueError: - print(f"error: --budget must be an integer", file=sys.stderr) - sys.exit(1) - i += 2 - elif args[i].startswith("--budget="): - try: - budget = int(args[i].split("=", 1)[1]) - except ValueError: - print(f"error: --budget must be an integer", file=sys.stderr) - sys.exit(1) - i += 1 - elif args[i] == "--context" and i + 1 < len(args): - context_filters.append(args[i + 1]) - i += 2 - elif args[i].startswith("--context="): - context_filters.append(args[i].split("=", 1)[1]) - i += 1 - elif args[i] == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - i += 2 - else: - i += 1 - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - if not gp.suffix == ".json": - print(f"error: graph file must be a .json file", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - try: - import json as _json - import networkx as _nx - - _raw = _json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - try: - from graphify.build import graph_has_legacy_ids as _legacy - if _legacy(_raw.get("nodes", [])): - print( - "[graphify] note: this graph uses the pre-#1504 node-ID scheme; " - "rebuild with `graphify extract --force` to get path-qualified IDs " - "(fixes same-name-file collisions).", - file=sys.stderr, - ) - except Exception: - pass - except Exception as exc: - print(f"error: could not load graph: {exc}", file=sys.stderr) - sys.exit(1) - import time as _time - _t0 = _time.perf_counter() - _mode = "dfs" if use_dfs else "bfs" - _result = _query_graph_text( - G, - question, - mode=_mode, - depth=2, - token_budget=budget, - context_filters=context_filters, - ) - querylog.log_query( - kind="query", - question=question, - corpus=str(gp), - result=_result, - mode=_mode, - depth=2, - token_budget=budget, - duration_ms=(_time.perf_counter() - _t0) * 1000, - ) - print(_result) - elif cmd == "affected": - if len(sys.argv) < 3: - print("Usage: graphify affected \"\" [--relation R] [--depth N] [--graph path]", file=sys.stderr) - sys.exit(1) - from graphify.affected import DEFAULT_AFFECTED_RELATIONS, format_affected, load_graph - query = sys.argv[2] - graph_path = _default_graph_path() - depth = 2 - relations: list[str] = [] - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - i += 2 - elif args[i].startswith("--graph="): - graph_path = args[i].split("=", 1)[1] - i += 1 - elif args[i] == "--depth" and i + 1 < len(args): - try: - depth = int(args[i + 1]) - except ValueError: - print("error: --depth must be an integer", file=sys.stderr) - sys.exit(1) - i += 2 - elif args[i].startswith("--depth="): - try: - depth = int(args[i].split("=", 1)[1]) - except ValueError: - print("error: --depth must be an integer", file=sys.stderr) - sys.exit(1) - i += 1 - elif args[i] == "--relation" and i + 1 < len(args): - relations.append(args[i + 1]) - i += 2 - elif args[i].startswith("--relation="): - relations.append(args[i].split("=", 1)[1]) - i += 1 - else: - i += 1 - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - if not gp.suffix == ".json": - print("error: graph file must be a .json file", file=sys.stderr) - sys.exit(1) - try: - graph = load_graph(gp) - except Exception as exc: - print(f"error: could not load graph: {exc}", file=sys.stderr) - sys.exit(1) - print( - format_affected( - graph, - query, - relations=relations or DEFAULT_AFFECTED_RELATIONS, - depth=depth, - ) - ) - elif cmd == "save-result": - # graphify save-result --question Q --answer A [--type T] [--nodes N1 N2 ...] - # [--outcome useful|dead_end|corrected] [--correction TEXT] - import argparse as _ap - - p = _ap.ArgumentParser(prog="graphify save-result") - p.add_argument("--question", required=True) - p.add_argument("--answer", default=None) - p.add_argument("--answer-file", dest="answer_file", default=None) - p.add_argument("--type", dest="query_type", default="query") - p.add_argument("--nodes", nargs="*", default=[]) - p.add_argument("--outcome", choices=("useful", "dead_end", "corrected"), default=None) - p.add_argument("--correction", default=None) - p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) - opts = p.parse_args(sys.argv[2:]) - if opts.answer_file: - opts.answer = Path(opts.answer_file).read_text(encoding="utf-8").strip() - elif not opts.answer: - p.error("--answer or --answer-file is required") - from graphify.ingest import save_query_result as _sqr - - out = _sqr( - question=opts.question, - answer=opts.answer, - memory_dir=Path(opts.memory_dir), - query_type=opts.query_type, - source_nodes=opts.nodes or None, - outcome=opts.outcome, - correction=opts.correction, - ) - print(f"Saved to {out}") - elif cmd == "reflect": - import argparse as _ap - - p = _ap.ArgumentParser(prog="graphify reflect") - p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) - p.add_argument( - "--out", - default=str(Path(_GRAPHIFY_OUT) / "reflections" / "LESSONS.md"), - ) - p.add_argument("--graph", default=None) - p.add_argument("--analysis", default=None) - p.add_argument("--labels", default=None) - p.add_argument("--half-life-days", type=float, default=30.0, - help="signal weight halves every N days (default 30)") - p.add_argument("--min-corroboration", type=int, default=2, - help="distinct useful results to promote a node to preferred (default 2)") - p.add_argument("--if-stale", action="store_true", - help="skip when LESSONS.md is already newer than every input " - "(e.g. the git hook just refreshed it)") - opts = p.parse_args(sys.argv[2:]) - from graphify.reflect import reflect as _reflect, lessons_fresh as _lessons_fresh - - graph_arg = opts.graph - if graph_arg is None: - default_graph = Path(_GRAPHIFY_OUT) / "graph.json" - if default_graph.exists(): - graph_arg = str(default_graph) - - _gp = Path(graph_arg) if graph_arg else None - _analysis_path = None - _labels_path = None - if _gp is not None: - _analysis_path = Path(opts.analysis) if opts.analysis else ( - _gp.parent / ".graphify_analysis.json") - _labels_path = Path(opts.labels) if opts.labels else ( - _gp.parent / ".graphify_labels.json") - - if opts.if_stale and _lessons_fresh( - Path(opts.out), Path(opts.memory_dir), _gp, _analysis_path, _labels_path - ): - print(f"Lessons already up to date -> {opts.out} (skipped; omit --if-stale to force)") - else: - out_path, agg = _reflect( - memory_dir=Path(opts.memory_dir), - out_path=Path(opts.out), - graph_path=_gp, - analysis_path=_analysis_path, - labels_path=_labels_path, - half_life_days=opts.half_life_days, - min_corroboration=opts.min_corroboration, - ) - c = agg["counts"] - print( - f"Reflected {agg['total']} memories " - f"({c['useful']} useful, {c['dead_end']} dead ends, " - f"{c['corrected']} corrected) -> {out_path}" - ) - elif cmd == "path": - if len(sys.argv) < 4: - print( - 'Usage: graphify path "" "" [--graph path]', - file=sys.stderr, - ) - sys.exit(1) - from graphify.serve import _score_nodes - from networkx.readwrite import json_graph - import networkx as _nx - - source_label = sys.argv[2] - target_label = sys.argv[3] - graph_path = _default_graph_path() - args = sys.argv[4:] - for i, a in enumerate(args): - if a == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - _raw = json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - # Force directed so the renderer can recover stored caller→callee direction. - _raw = {**_raw, "directed": True} - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - src_scored = _score_nodes(G, [t.lower() for t in source_label.split()]) - tgt_scored = _score_nodes(G, [t.lower() for t in target_label.split()]) - if not src_scored: - print(f"No node matching '{source_label}' found.", file=sys.stderr) - sys.exit(1) - if not tgt_scored: - print(f"No node matching '{target_label}' found.", file=sys.stderr) - sys.exit(1) - src_nid, tgt_nid = src_scored[0][1], tgt_scored[0][1] - # Ambiguity guard: when both queries resolve to the same node, the - # shortest path is trivially zero hops, which is almost never what the - # caller wanted (see bug #828). - if src_nid == tgt_nid: - print( - f"'{source_label}' and '{target_label}' both resolved to the same " - f"node '{src_nid}'. Use a more specific label or the exact node ID.", - file=sys.stderr, - ) - sys.exit(1) - for _name, _scored in (("source", src_scored), ("target", tgt_scored)): - if len(_scored) >= 2: - _top, _runner = _scored[0][0], _scored[1][0] - if _top > 0 and (_top - _runner) / _top < 0.10: - print( - f"warning: {_name} match was ambiguous " - f"(top score {_top:g}, runner-up {_runner:g})", - file=sys.stderr, - ) - try: - path_nodes = _nx.shortest_path(G.to_undirected(as_view=True), src_nid, tgt_nid) - except (_nx.NetworkXNoPath, _nx.NodeNotFound): - print(f"No path found between '{source_label}' and '{target_label}'.") - sys.exit(0) - hops = len(path_nodes) - 1 - segments = [] - from graphify.build import edge_data - for i in range(len(path_nodes) - 1): - u, v = path_nodes[i], path_nodes[i + 1] - # Check which direction the stored edge points. - if G.has_edge(u, v): - edata = edge_data(G, u, v) - forward = True - else: - edata = edge_data(G, v, u) - forward = False - rel = edata.get("relation", "") - conf = edata.get("confidence", "") - conf_str = f" [{conf}]" if conf else "" - if i == 0: - segments.append(G.nodes[u].get("label", u)) - if forward: - segments.append(f"--{rel}{conf_str}--> {G.nodes[v].get('label', v)}") - else: - segments.append(f"<--{rel}{conf_str}-- {G.nodes[v].get('label', v)}") - print(f"Shortest path ({hops} hops):\n " + " ".join(segments)) - from graphify import querylog - querylog.log_query( - kind="path", - question=f"{sys.argv[2]} -> {sys.argv[3]}", - corpus=str(gp), - nodes_returned=hops, - ) - - elif cmd == "explain": - if len(sys.argv) < 3: - print('Usage: graphify explain "" [--graph path]', file=sys.stderr) - sys.exit(1) - from graphify.serve import _find_node - from networkx.readwrite import json_graph - - label = sys.argv[2] - graph_path = _default_graph_path() - args = sys.argv[3:] - for i, a in enumerate(args): - if a == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - _raw = json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - # Force directed so the renderer can recover stored caller→callee direction. - _raw = {**_raw, "directed": True} - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - matches = _find_node(G, label) - if not matches: - print(f"No node matching '{label}' found.") - sys.exit(0) - nid = matches[0] - d = G.nodes[nid] - print(f"Node: {d.get('label', nid)}") - print(f" ID: {nid}") - print( - f" Source: {d.get('source_file', '')} {d.get('source_location', '')}".rstrip() - ) - print(f" Type: {d.get('file_type', '')}") - print(f" Community: {d.get('community_name') or d.get('community', '')}") - # Work-memory overlay: a derived experiential hint from `graphify reflect`, - # merged in display-only from the .graphify_learning.json sidecar next to - # graph.json. No line when the node has no overlay entry. - try: - from graphify.reflect import load_learning_overlay as _llo - from graphify.security import sanitize_label as _sl - _overlay = _llo(gp) - _entry = _overlay.get(str(nid)) - if _entry: - _status = _sl(str(_entry.get("status", ""))) - if _status == "contested": - _line = (f" Lesson: contested (useful {_entry.get('uses', 0)} / " - f"dead-end {_entry.get('neg', 0)})") - elif _status == "preferred": - _line = (f" Lesson: preferred source (start here) — " - f"{_entry.get('uses', 0)} useful, score={_entry.get('score', 0)}") - else: - _line = (f" Lesson: {_status or 'tentative'} — " - f"{_entry.get('uses', 0)} useful, score={_entry.get('score', 0)}") - if _entry.get("stale"): - _line += " [code changed since — re-verify]" - print(_line) - except Exception: - pass - print(f" Degree: {G.degree(nid)}") - from graphify.build import edge_data - connections: list[tuple[str, str, dict]] = [] # (direction, neighbor_id, edge_data) - for nb in G.successors(nid): - connections.append(("out", nb, edge_data(G, nid, nb))) - for nb in G.predecessors(nid): - connections.append(("in", nb, edge_data(G, nb, nid))) - if connections: - print(f"\nConnections ({len(connections)}):") - connections.sort(key=lambda c: G.degree(c[1]), reverse=True) - for direction, nb, edata in connections[:20]: - rel = edata.get("relation", "") - conf = edata.get("confidence", "") - arrow = "-->" if direction == "out" else "<--" - print(f" {arrow} {G.nodes[nb].get('label', nb)} [{rel}] [{conf}]") - if len(connections) > 20: - print(f" ... and {len(connections) - 20} more") - from graphify import querylog - querylog.log_query( - kind="explain", - question=sys.argv[2], - corpus=str(gp), - nodes_returned=len(connections), - ) - - elif cmd == "diagnose": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd != "multigraph": - print( - "Usage: graphify diagnose multigraph " - "[--graph path] [--json] [--max-examples N] " - "[--directed] [--undirected] [--extract-path path]", - file=sys.stderr, - ) - sys.exit(1) - - graph_path = Path(_default_graph_path()) - max_examples = 5 - directed: bool | None = None - direction_flag: str | None = None - json_output = False - extract_path: Path | None = None - - i = 3 - while i < len(sys.argv): - arg = sys.argv[i] - if arg == "--graph": - i += 1 - if i >= len(sys.argv): - print("error: --graph requires a path", file=sys.stderr) - sys.exit(1) - graph_path = Path(sys.argv[i]) - elif arg == "--json": - json_output = True - elif arg == "--max-examples": - i += 1 - if i >= len(sys.argv): - print("error: --max-examples requires an integer", file=sys.stderr) - sys.exit(1) - try: - max_examples = int(sys.argv[i]) - except ValueError: - print("error: --max-examples requires an integer", file=sys.stderr) - sys.exit(1) - if max_examples < 0: - print("error: --max-examples must be >= 0", file=sys.stderr) - sys.exit(1) - elif arg == "--directed": - if direction_flag == "undirected": - print( - "error: --directed and --undirected are mutually exclusive", - file=sys.stderr, - ) - sys.exit(1) - direction_flag = "directed" - directed = True - elif arg == "--undirected": - if direction_flag == "directed": - print( - "error: --directed and --undirected are mutually exclusive", - file=sys.stderr, - ) - sys.exit(1) - direction_flag = "undirected" - directed = False - elif arg == "--extract-path": - i += 1 - if i >= len(sys.argv): - print("error: --extract-path requires a path", file=sys.stderr) - sys.exit(1) - extract_path = Path(sys.argv[i]) - else: - print(f"error: unknown diagnose option {arg}", file=sys.stderr) - sys.exit(1) - i += 1 - - from graphify.diagnostics import ( - diagnose_file, - format_diagnostic_json, - format_diagnostic_report, - ) - - try: - summary = diagnose_file( - graph_path, - directed=directed, - root=Path(".").resolve(), - max_examples=max_examples, - extract_path=extract_path, - ) - except Exception as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - - if json_output: - print(json.dumps(format_diagnostic_json(summary), indent=2)) - else: - print(format_diagnostic_report(summary)) - - elif cmd == "add": - if len(sys.argv) < 3: - print( - "Usage: graphify add [--author Name] [--contributor Name] [--dir ./raw]", - file=sys.stderr, - ) - sys.exit(1) - from graphify.ingest import ingest as _ingest - - url = sys.argv[2] - author: str | None = None - contributor: str | None = None - target_dir = Path("raw") - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--author" and i + 1 < len(args): - author = args[i + 1] - i += 2 - elif args[i] == "--contributor" and i + 1 < len(args): - contributor = args[i + 1] - i += 2 - elif args[i] == "--dir" and i + 1 < len(args): - target_dir = Path(args[i + 1]) - i += 2 - else: - i += 1 - try: - saved = _ingest(url, target_dir, author=author, contributor=contributor) - print(f"Saved to {saved}") - print("Run /graphify --update in your AI assistant to update the graph.") - except Exception as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - - elif cmd == "watch": - watch_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(".") - if not watch_path.exists(): - print(f"error: path not found: {watch_path}", file=sys.stderr) - sys.exit(1) - from graphify.watch import watch as _watch - - try: - _watch(watch_path) - except ImportError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - - elif cmd in ("cluster-only", "label"): - # `label` is `cluster-only` that always (re)generates community names with - # the configured backend, even when a .graphify_labels.json already exists. - force_relabel = cmd == "label" - # Mirror the tree/export arg-parsing pattern: walk argv so flags and - # the optional positional path can appear in any order (#724). - no_viz = "--no-viz" in sys.argv - no_label = "--no-label" in sys.argv - missing_only = "--missing-only" in sys.argv - co_timing = "--timing" in sys.argv - _backend_arg = next((a for a in sys.argv if a.startswith("--backend=")), None) - label_backend = _backend_arg.split("=", 1)[1] if _backend_arg else None - _model_arg = next((a for a in sys.argv if a.startswith("--model=")), None) - label_model = _model_arg.split("=", 1)[1] if _model_arg else None - _min_cs_arg = next((a for a in sys.argv if a.startswith("--min-community-size=")), None) - min_community_size = int(_min_cs_arg.split("=")[1]) if _min_cs_arg else 3 - args = sys.argv[2:] - watch_path: Path | None = None - graph_override: Path | None = None - co_resolution: float = 1.0 - co_exclude_hubs: float | None = None - label_max_concurrency: int = 4 - label_batch_size: int = 100 - i_arg = 0 - while i_arg < len(args): - a = args[i_arg] - if a == "--graph" and i_arg + 1 < len(args): - graph_override = Path(args[i_arg + 1]); i_arg += 2 - elif a == "--backend" and i_arg + 1 < len(args): - label_backend = args[i_arg + 1]; i_arg += 2 - elif a.startswith("--backend="): - label_backend = a.split("=", 1)[1]; i_arg += 1 - elif a == "--model" and i_arg + 1 < len(args): - label_model = args[i_arg + 1]; i_arg += 2 - elif a.startswith("--model="): - label_model = a.split("=", 1)[1]; i_arg += 1 - elif a == "--resolution" and i_arg + 1 < len(args): - co_resolution = float(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--resolution="): - co_resolution = float(a.split("=", 1)[1]); i_arg += 1 - elif a == "--exclude-hubs" and i_arg + 1 < len(args): - co_exclude_hubs = float(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--exclude-hubs="): - co_exclude_hubs = float(a.split("=", 1)[1]); i_arg += 1 - elif a == "--max-concurrency" and i_arg + 1 < len(args): - label_max_concurrency = int(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--max-concurrency="): - label_max_concurrency = int(a.split("=", 1)[1]); i_arg += 1 - elif a == "--batch-size" and i_arg + 1 < len(args): - label_batch_size = int(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--batch-size="): - label_batch_size = int(a.split("=", 1)[1]); i_arg += 1 - elif a in ("--no-viz", "--missing-only") or a.startswith("--min-community-size="): - i_arg += 1 - elif a.startswith("--"): - i_arg += 1 - elif watch_path is None: - watch_path = Path(a); i_arg += 1 - else: - i_arg += 1 - if watch_path is None: - watch_path = Path(".") - graph_json = graph_override if graph_override is not None else watch_path / _GRAPHIFY_OUT / "graph.json" - if not graph_json.exists(): - print( - f"error: no graph found at {graph_json} — run /graphify first", - file=sys.stderr, - ) - sys.exit(1) - from networkx.readwrite import json_graph as _jg - from graphify.build import build_from_json - from graphify.cluster import cluster, score_all, remap_communities_to_previous - from graphify.analyze import ( - god_nodes, - surprising_connections, - suggest_questions, - ) - from graphify.report import generate - from graphify.export import to_json, to_html - - stages = _StageTimer(co_timing) - print("Loading existing graph...") - # Solution 3 (#1019): don't hard-exit on an oversized graph.json here. - # Core outputs (graph.json + GRAPH_REPORT.md) still get written; the - # graph.html render below falls back to the community-aggregation view - # (node_limit=5000) when over the cap. - from graphify.security import check_graph_file_size_cap as _check_cap - _over_cap = False - try: - _check_cap(graph_json) - except ValueError: - _over_cap = True - try: - _over_cap_bytes = graph_json.stat().st_size - except OSError: - _over_cap_bytes = -1 - print( - f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " - f"falling back to community-aggregation view (node_limit=5000)", - file=sys.stderr, - ) - _raw = json.loads(graph_json.read_text(encoding="utf-8")) - _directed = bool(_raw.get("directed", False)) - G = build_from_json(_raw, directed=_directed) - print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") - stages.mark("load") - print("Re-clustering...") - communities = cluster(G, resolution=co_resolution, exclude_hubs_percentile=co_exclude_hubs) - # Mirror the watch/update path (#822): map new cids to prior ones by - # node-overlap so the existing .graphify_labels.json keeps attaching - # to the same conceptual community after re-clustering. Without this, - # labels follow raw cid index and become misaligned whenever the - # graph has changed between labeling and cluster-only (#1027). - previous_node_community = { - n["id"]: n["community"] - for n in _raw.get("nodes", []) - if n.get("community") is not None and n.get("id") is not None - } - if previous_node_community: - communities = remap_communities_to_previous(communities, previous_node_community) - stages.mark("cluster") - cohesion = score_all(G, communities) - gods = god_nodes(G) - surprises = surprising_connections(G, communities) - stages.mark("analyze") - out = watch_path / _GRAPHIFY_OUT - out.mkdir(parents=True, exist_ok=True) - labels_path = out / ".graphify_labels.json" - existing_labels: dict[int, str] = {} - if labels_path.exists(): - try: - existing_labels = { - int(k): v - for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items() - if isinstance(v, str) - } - except Exception: - existing_labels = {} - # Accumulate token usage from the labeling LLM calls so cluster-only mode - # reports real cost instead of a hardcoded zero (#1694). Stays {0, 0} on - # the reuse / no-label paths, which make no LLM calls. - label_token_usage = {"input": 0, "output": 0} - if labels_path.exists() and not force_relabel: - # Reuse saved labels, but don't blindly trust them: the graph may have - # been re-scoped/re-clustered since labeling, in which case a cid now - # covers a DIFFERENT community and its old (LLM) name is wrong (#label-stale). - # Validate each community against the membership signature saved beside the - # labels; any community that changed (or has no saved label) is renamed by - # its current hub — deterministic and correct-by-construction — and the user - # is told to `graphify label` for fresh LLM names. Unchanged communities keep - # their saved label. When no signature sidecar exists (labels predate this), - # fall back to hub-filling only the communities missing a label. - from graphify.cluster import community_member_sigs, label_communities_by_hub - sig_path = labels_path.parent / (labels_path.name + ".sig") - saved_sigs: dict[int, str] = {} - if sig_path.exists(): - try: - saved_sigs = { - int(k): v for k, v in - json.loads(sig_path.read_text(encoding="utf-8")).items() - if isinstance(v, str) - } - except Exception: - saved_sigs = {} - cur_sigs = community_member_sigs(communities) - count_mismatch = len(existing_labels) != len(communities) - labels = {} - hub_labels: dict[int, str] | None = None - changed = 0 - for cid in communities: - have_label = cid in existing_labels - if saved_sigs: - # Precise: the membership signature tells us if this exact - # community changed since it was labeled. - fresh = have_label and saved_sigs.get(cid) == cur_sigs.get(cid) - else: - # No signature sidecar (labels predate it). A differing community - # COUNT means the labels describe a different clustering, so a cid's - # old label can't be trusted; equal count is the best "same" signal. - fresh = have_label and not count_mismatch - if fresh: - labels[cid] = existing_labels[cid] - else: - if hub_labels is None: - hub_labels = label_communities_by_hub(G, communities) - labels[cid] = hub_labels[cid] - if have_label: - changed += 1 - if changed: - print( - f"[graphify] community set changed since labeling " - f"({len(existing_labels)} saved labels, {len(communities)} communities now; " - f"renamed {changed} community(ies) by their hub). " - f"Run `graphify label` to refresh names with the LLM.", - file=sys.stderr, - ) - elif no_label and not force_relabel: - labels = {cid: f"Community {cid}" for cid in communities} - else: - # No labels file yet (or `graphify label` forced a refresh). When run - # standalone there is no orchestrating agent to do skill.md Step 5, so - # auto-name communities rather than leave "Community N" (#1097). - from graphify.cluster import label_communities_by_hub - from graphify.llm import generate_community_labels - print("Labeling communities...") - # Deterministic, LLM-free base labels: name each community after its - # highest-degree hub, so the report is readable even with no backend - # (previously bare "Community N"). A configured LLM backend overrides these - # with richer names below; its no-backend placeholder fallback does NOT. - hub_labels = label_communities_by_hub(G, communities) - label_communities_input = communities - labels = dict(hub_labels) - if missing_only: - labels = { - cid: existing_labels.get(cid, hub_labels[cid]) - for cid in communities - } - label_communities_input = { - cid: members - for cid, members in communities.items() - if cid not in existing_labels or existing_labels.get(cid) == f"Community {cid}" - } - generated_labels, _ = generate_community_labels( - G, label_communities_input, backend=label_backend, model=label_model, gods=gods, - max_concurrency=label_max_concurrency, batch_size=label_batch_size, - usage_out=label_token_usage, - ) - # Only let the LLM OVERRIDE where it produced a real name — its no-backend - # fallback returns "Community {cid}" placeholders, which must not clobber - # the deterministic hub labels. - labels.update({ - cid: v for cid, v in generated_labels.items() - if v and v != f"Community {cid}" - }) - stages.mark("label") - questions = suggest_questions(G, communities, labels) - tokens = label_token_usage - from graphify.export import _git_head as _gh - _commit = _gh() - from graphify.report import load_learning_for_report as _llfr - report = generate(G, communities, cohesion, labels, gods, surprises, - {"warning": "cluster-only mode — file stats not available"}, - tokens, str(watch_path), suggested_questions=questions, - min_community_size=min_community_size, built_at_commit=_commit, - learning=_llfr(out / "graph.json")) - (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") - stages.mark("report") - from graphify.export import backup_if_protected as _backup - _backup(out) - analysis = { - "communities": {str(k): v for k, v in communities.items()}, - "cohesion": {str(k): v for k, v in cohesion.items()}, - "gods": gods, - "surprises": surprises, - "questions": questions, - } - (out / ".graphify_analysis.json").write_text( - json.dumps(analysis, indent=2, ensure_ascii=False), - encoding="utf-8", - ) - to_json(G, communities, str(out / "graph.json"), community_labels=labels) - labels_path.write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding="utf-8") - # Membership signatures beside the labels so a later cluster-only can detect - # which communities changed and avoid reusing a stale label (see reuse above). - from graphify.cluster import community_member_sigs as _cms - (labels_path.parent / (labels_path.name + ".sig")).write_text( - json.dumps({str(k): v for k, v in _cms(communities).items()}), encoding="utf-8") - - # Mirror watch.py pattern: gate to_html so core outputs (graph.json + - # GRAPH_REPORT.md) always land. Honor --no-viz explicitly; otherwise - # fall back to ValueError handling so an oversized graph doesn't crash - # the CLI mid-write and leave a stale graph.html on disk. - html_target = out / "graph.html" - if no_viz: - if html_target.exists(): - html_target.unlink() - stages.mark("export"); stages.total() - print(f"Done - {len(communities)} communities. GRAPH_REPORT.md and graph.json updated (--no-viz; graph.html removed).") - else: - try: - # Over-cap fallback (#1019): force the community-aggregation - # path so an oversized graph still renders a usable graph.html. - _node_limit = 5000 if _over_cap else None - to_html(G, communities, str(html_target), community_labels=labels or None, - node_limit=_node_limit) - stages.mark("export"); stages.total() - print(f"Done - {len(communities)} communities. GRAPH_REPORT.md, graph.json and graph.html updated.") - except ValueError as viz_err: - if html_target.exists(): - html_target.unlink() - print(f"Skipped graph.html: {viz_err}") - stages.mark("export"); stages.total() - print(f"Done - {len(communities)} communities. GRAPH_REPORT.md and graph.json updated.") - - elif cmd == "update": - force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") - no_cluster = False - args = sys.argv[2:] - watch_arg: str | None = None - for a in args: - if a == "--force": - force = True - continue - if a == "--no-cluster": - no_cluster = True - continue - if a.startswith("-"): - print(f"error: unknown update option: {a}", file=sys.stderr) - sys.exit(2) - if watch_arg is not None: - print("error: update accepts at most one path argument", file=sys.stderr) - sys.exit(2) - watch_arg = a - - if watch_arg is not None: - watch_path = Path(watch_arg) - else: - # Try to recover the scan root saved by the last full build - saved = Path(_GRAPHIFY_OUT) / ".graphify_root" - if saved.exists(): - watch_path = Path(saved.read_text(encoding="utf-8").strip()) - else: - watch_path = Path(".") - if not watch_path.exists(): - print(f"error: path not found: {watch_path}", file=sys.stderr) - sys.exit(1) - from graphify.watch import _rebuild_code - - print(f"Re-extracting code files in {watch_path} (no LLM needed)...") - # Interactive CLI: block on the per-repo lock rather than skip, so the - # user sees their explicit `graphify update` complete instead of - # exiting silently when a hook-driven rebuild happens to be running. - ok = _rebuild_code(watch_path, force=force, no_cluster=no_cluster, block_on_lock=True) - if ok: - print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.") - if not ( - os.environ.get("GEMINI_API_KEY") - or os.environ.get("GOOGLE_API_KEY") - or os.environ.get("MOONSHOT_API_KEY") - or os.environ.get("DEEPSEEK_API_KEY") - or os.environ.get("GRAPHIFY_NO_TIPS") - ): - print("Tip: set GEMINI_API_KEY or GOOGLE_API_KEY to use Gemini for semantic extraction.") - else: - print( - "Nothing to update or rebuild failed — check output above.", - file=sys.stderr, - ) - sys.exit(1) - - elif cmd == "hook-check": - # Codex Desktop rejects hookSpecificOutput.additionalContext on PreToolUse. - # Keep this as a cross-platform no-op so installed hooks never break Bash - # tool calls. Graph guidance reaches the agent via AGENTS.md / skill instead. - sys.exit(0) - elif cmd == "hook-guard": - # Shell-agnostic Claude/Codebuddy PreToolUse guard (#522). Replaces the old - # inline-bash hooks that failed on Windows. Prints an additionalContext nudge - # toward graphify when a graph exists; always exits 0 (never blocks a tool). - _run_hook_guard(sys.argv[2] if len(sys.argv) > 2 else "") - sys.exit(0) - elif cmd == "check-update": - if len(sys.argv) < 3: - print("Usage: graphify check-update ", file=sys.stderr) - sys.exit(1) - from graphify.watch import check_update - - check_update(Path(sys.argv[2]).resolve()) - sys.exit(0) - elif cmd == "tree": - # Emit a D3 v7 collapsible-tree HTML view of graph.json: - # expand-all / collapse-all / reset-view buttons, multi-line - # wrapText labels with separately-coloured name + count, - # depth-based palette, click-to-toggle subtree, hover inspector - # showing top-K outbound edges per symbol. - from typing import Optional as _Opt - from graphify.tree_html import write_tree_html, DEFAULT_MAX_CHILDREN - graph_path = Path(_GRAPHIFY_OUT) / "graph.json" - output_path: "_Opt[Path]" = None - root: "_Opt[str]" = None - max_children = DEFAULT_MAX_CHILDREN - top_k_edges = 0 - project_label: "_Opt[str]" = None - args = sys.argv[2:] - i_arg = 0 - while i_arg < len(args): - a = args[i_arg] - if a == "--graph" and i_arg + 1 < len(args): - graph_path = Path(args[i_arg + 1]); i_arg += 2 - elif a == "--output" and i_arg + 1 < len(args): - output_path = Path(args[i_arg + 1]); i_arg += 2 - elif a == "--root" and i_arg + 1 < len(args): - root = args[i_arg + 1]; i_arg += 2 - elif a == "--max-children" and i_arg + 1 < len(args): - max_children = int(args[i_arg + 1]); i_arg += 2 - elif a == "--top-k-edges" and i_arg + 1 < len(args): - top_k_edges = int(args[i_arg + 1]); i_arg += 2 - elif a == "--label" and i_arg + 1 < len(args): - project_label = args[i_arg + 1]; i_arg += 2 - elif a in ("-h", "--help"): - print("Usage: graphify tree [--graph PATH] [--output HTML]") - print(" --graph PATH path to graph.json (default graphify-out/graph.json)") - print(" --output HTML output path (default graphify-out/GRAPH_TREE.html)") - print(" --root PATH filesystem root (default: longest common dir of all source_files)") - print(" --max-children N cap visible children per node (default 200)") - print(" --top-k-edges N pre-compute top-K outbound edges per symbol (default 12)") - print(" --label NAME project label shown in the page header") - return - else: - i_arg += 1 - if not graph_path.is_file(): - print(f"error: graph.json not found at {graph_path}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(graph_path) - if output_path is None: - output_path = graph_path.parent / "GRAPH_TREE.html" - out = write_tree_html( - graph_path=graph_path, output_path=output_path, - root=root, max_children=max_children, - top_k_edges=top_k_edges, project_label=project_label, - ) - size_kb = out.stat().st_size / 1024 - print(f"wrote {out} ({size_kb:.1f} KB)") - print(f"open with: xdg-open {out} (or file://{out.resolve()})") - sys.exit(0) - - elif cmd == "merge-driver": - # git merge driver for graph.json — takes (base, current, other) and writes - # the union of current+other nodes/edges back to current. Exits 1 on - # corrupt input so git surfaces the conflict instead of silently - # accepting a poisoned merge (see F-005). - # Usage: graphify merge-driver %O %A %B (set in .git/config merge driver) - if len(sys.argv) < 5: - print("Usage: graphify merge-driver ", file=sys.stderr) - sys.exit(1) - _base_path, _current_path, _other_path = sys.argv[2], sys.argv[3], sys.argv[4] - # Hard caps so a malicious or corrupted graph.json cannot exhaust memory - # at parse time. 50 MB / 100k nodes are well above any realistic graph - # (typical graphs are <5 MB / <50k nodes); anything larger should fail - # the merge so a human can investigate. - _MERGE_MAX_BYTES = 50 * 1024 * 1024 - _MERGE_MAX_NODES = 100_000 - import networkx as _nx - from networkx.readwrite import json_graph as _jg - def _load_graph(p: str): - path_obj = Path(p) - try: - size = path_obj.stat().st_size - except OSError as exc: - raise RuntimeError(f"cannot stat {p}: {exc}") from exc - if size > _MERGE_MAX_BYTES: - raise RuntimeError( - f"graph.json {p} is {size} bytes, exceeds {_MERGE_MAX_BYTES}-byte cap" - ) - data = json.loads(path_obj.read_text(encoding="utf-8")) - try: - return _jg.node_link_graph(data, edges="links"), data - except TypeError: - return _jg.node_link_graph(data), data - try: - G_cur, _ = _load_graph(_current_path) - G_oth, _ = _load_graph(_other_path) - except Exception as exc: - print(f"[graphify merge-driver] error loading graphs: {exc}", file=sys.stderr) - sys.exit(1) # surface the conflict so git doesn't accept a corrupt merge - merged = _nx.compose(G_cur, G_oth) - if merged.number_of_nodes() > _MERGE_MAX_NODES: - print( - f"[graphify merge-driver] merged graph has {merged.number_of_nodes()} nodes, " - f"exceeds {_MERGE_MAX_NODES}-node cap; aborting merge.", - file=sys.stderr, - ) - sys.exit(1) - try: - out_data = _jg.node_link_data(merged, edges="links") - except TypeError: - out_data = _jg.node_link_data(merged) - Path(_current_path).write_text(json.dumps(out_data, indent=2), encoding="utf-8") - sys.exit(0) - - elif cmd == "merge-graphs": - # graphify merge-graphs graph1.json graph2.json ... --out merged.json - args = sys.argv[2:] - graph_paths: list[Path] = [] - out_path = Path(_GRAPHIFY_OUT) / "merged-graph.json" - i = 0 - while i < len(args): - if args[i] == "--out" and i + 1 < len(args): - out_path = Path(args[i + 1]) - i += 2 - else: - graph_paths.append(Path(args[i])) - i += 1 - if len(graph_paths) < 2: - print( - "Usage: graphify merge-graphs [...] [--out merged.json]", - file=sys.stderr, - ) - sys.exit(1) - import networkx as _nx - from networkx.readwrite import json_graph as _jg - from graphify.build import prefix_graph_for_global as _prefix, distinct_repo_tags as _repo_tags - graphs = [] - for gp in graph_paths: - if not gp.exists(): - print(f"error: not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - data = json.loads(gp.read_text(encoding="utf-8")) - # Normalize edges/links key before loading — graphify writes "links" - # via node_link_data but older runs may have used "edges" (#738). - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - try: - G = _jg.node_link_graph(data, edges="links") - except TypeError: - G = _jg.node_link_graph(data) - graphs.append(G) - # nx.compose requires all graphs to be the same type. When input graphs - # come from different sources (e.g. an AST-only run vs a full LLM run) one - # may be a MultiGraph and another a Graph. Normalise everything to Graph - # (the graphify default) by converting MultiGraphs with nx.Graph(). - def _to_simple(g: "_nx.Graph") -> "_nx.Graph": - # nx.compose requires every graph to be the same type. Inputs may - # disagree on BOTH axes — directed vs undirected, and multi vs simple - # — because per-repo graph.json files are written by different extract - # paths at different times. Normalise everything to a plain undirected - # Graph (the merged cross-repo view is undirected anyway), which covers - # DiGraph / MultiGraph / MultiDiGraph. Without this a directed input - # crashed compose with "All graphs must be directed or undirected" (#1606). - if type(g) is not _nx.Graph: - return _nx.Graph(g) - return g - # Unique repo tag per graph. The bare `graphify-out/..` dir name is not - # unique across inputs (src/graphify-out and frontend/src/graphify-out both - # → "src"), which collides same-stem node ids and silently merges unrelated - # entities (#1729). distinct_repo_tags guarantees a distinct prefix per graph. - repo_tags = _repo_tags(graph_paths) - naive_tags = [gp.parent.parent.name for gp in graph_paths] - if len(set(naive_tags)) != len(naive_tags): - print(f" note: repo dir names collide; using distinct tags: {', '.join(repo_tags)}") - merged = _nx.Graph() - for G, repo_tag in zip(graphs, repo_tags): - prefixed = _to_simple(_prefix(G, repo_tag)) - merged = _nx.compose(merged, prefixed) - try: - out_data = _jg.node_link_data(merged, edges="links") - except TypeError: - out_data = _jg.node_link_data(merged) - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(out_data, indent=2), encoding="utf-8") - print(f"Merged {len(graphs)} graphs -> {merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges") - print(f"Written to: {out_path}") - - elif cmd == "clone": - if len(sys.argv) < 3: - print( - "Usage: graphify clone [--branch ] [--out ]", - file=sys.stderr, - ) - sys.exit(1) - url = sys.argv[2] - branch: str | None = None - out_dir: Path | None = None - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--branch" and i + 1 < len(args): - branch = args[i + 1] - i += 2 - elif args[i] == "--out" and i + 1 < len(args): - out_dir = Path(args[i + 1]) - i += 2 - else: - i += 1 - local_path = _clone_repo(url, branch=branch, out_dir=out_dir) - print(local_path) - - elif cmd == "export": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd not in ("html", "callflow-html", "obsidian", "wiki", "svg", "graphml", "neo4j", "falkordb"): - print("Usage: graphify export ", file=sys.stderr) - print(" html [--graph PATH] [--labels PATH] [--node-limit N] [--no-viz]", file=sys.stderr) - print(" callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH] [--report PATH] [--sections PATH] [--output HTML]", file=sys.stderr) - print(" [--lang auto|zh-CN|en] [--max-sections N] [--diagram-scale N]", file=sys.stderr) - print(" obsidian [--graph PATH] [--labels PATH] [--dir PATH]", file=sys.stderr) - print(" wiki [--graph PATH] [--labels PATH]", file=sys.stderr) - print(" svg [--graph PATH] [--labels PATH]", file=sys.stderr) - print(" graphml [--graph PATH]", file=sys.stderr) - print(" neo4j [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) - print(" (or set NEO4J_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) - print(" falkordb [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) - print(" (or set FALKORDB_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) - sys.exit(1) - - # Parse shared args - args = sys.argv[3:] - graph_path = Path(_GRAPHIFY_OUT) / "graph.json" - graph_path_explicit = False - labels_path = Path(_GRAPHIFY_OUT) / ".graphify_labels.json" - labels_path_explicit = False - report_path = Path(_GRAPHIFY_OUT) / "GRAPH_REPORT.md" - report_path_explicit = False - sections_path: Path | None = None - callflow_output: Path | None = None - callflow_lang = "auto" - callflow_max_sections = 15 - callflow_diagram_scale = 1.0 - callflow_max_diagram_nodes = 18 - callflow_max_diagram_edges = 24 - analysis_path = Path(_GRAPHIFY_OUT) / ".graphify_analysis.json" - node_limit = 5000 - no_viz = False - obsidian_dir = Path(_GRAPHIFY_OUT) / "obsidian" - # Shared push-connection settings for the graph-database sinks (neo4j, - # falkordb), parsed from the generic --push/--user/--password flags below. - push_uri: str | None = None - push_user = "neo4j" # Neo4j default user; FalkorDB auth is optional and ignores it - # F-031: prefer an env var so the password never appears on argv (visible - # in `ps` output / shell history). The explicit --password flag still - # overrides it. Each sink reads its own var: FALKORDB_PASSWORD for falkordb, - # NEO4J_PASSWORD otherwise. - push_password: str | None = ( - os.environ.get("FALKORDB_PASSWORD") if subcmd == "falkordb" - else os.environ.get("NEO4J_PASSWORD") - ) or None - i = 0 - while i < len(args): - a = args[i] - if a == "--graph" and i + 1 < len(args): - graph_path = Path(args[i + 1]) - graph_path_explicit = True - i += 2 - elif a == "--labels" and i + 1 < len(args): - labels_path = Path(args[i + 1]) - labels_path_explicit = True - i += 2 - elif a == "--report" and i + 1 < len(args): - report_path = Path(args[i + 1]) - report_path_explicit = True - i += 2 - elif a == "--sections" and i + 1 < len(args): - sections_path = Path(args[i + 1]); i += 2 - elif a == "--output" and i + 1 < len(args): - callflow_output = Path(args[i + 1]).expanduser() - if not callflow_output.is_absolute(): - callflow_output = Path.cwd() / callflow_output - i += 2 - elif a == "--lang" and i + 1 < len(args): - callflow_lang = args[i + 1]; i += 2 - elif a == "--max-sections" and i + 1 < len(args): - callflow_max_sections = int(args[i + 1]); i += 2 - elif a == "--diagram-scale" and i + 1 < len(args): - callflow_diagram_scale = float(args[i + 1]); i += 2 - elif a == "--max-diagram-nodes" and i + 1 < len(args): - callflow_max_diagram_nodes = int(args[i + 1]); i += 2 - elif a == "--max-diagram-edges" and i + 1 < len(args): - callflow_max_diagram_edges = int(args[i + 1]); i += 2 - elif a in ("-h", "--help") and subcmd == "callflow-html": - print("Usage: graphify export callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH]") - print(" --report PATH path to GRAPH_REPORT.md") - print(" --sections PATH JSON section definitions") - print(" --output HTML output path (default graphify-out/-callflow.html)") - print(" --lang LANG auto, zh-CN, en, etc. (default auto)") - print(" --max-sections N maximum auto-derived sections (default 15)") - print(" --diagram-scale N Mermaid diagram scale (default 1.0)") - print(" --max-diagram-nodes N representative nodes per section (default 18)") - print(" --max-diagram-edges N representative edges per section (default 24)") - sys.exit(0) - elif a == "--node-limit" and i + 1 < len(args): - node_limit = int(args[i + 1]); i += 2 - elif a == "--no-viz": - no_viz = True; i += 1 - elif a == "--dir" and i + 1 < len(args): - obsidian_dir = Path(args[i + 1]); i += 2 - elif a == "--push" and i + 1 < len(args): - push_uri = args[i + 1]; i += 2 - elif a == "--user" and i + 1 < len(args): - push_user = args[i + 1]; i += 2 - elif a == "--password" and i + 1 < len(args): - push_password = args[i + 1]; i += 2 - elif subcmd == "callflow-html" and not a.startswith("-") and not graph_path_explicit: - candidate = Path(a) - if candidate.name == "graph.json" or candidate.suffix.lower() == ".json": - graph_path = candidate - elif (candidate / "graph.json").exists(): - graph_path = candidate / "graph.json" - else: - graph_path = candidate / _GRAPHIFY_OUT / "graph.json" - graph_path_explicit = True - i += 1 - else: - i += 1 - - graph_path = graph_path.expanduser() - if graph_path_explicit: - graph_out_dir = graph_path.parent - if not labels_path_explicit: - labels_path = graph_out_dir / ".graphify_labels.json" - if not report_path_explicit: - report_path = graph_out_dir / "GRAPH_REPORT.md" - labels_path = labels_path.expanduser() - report_path = report_path.expanduser() - - if not graph_path.exists(): - print(f"error: graph not found: {graph_path}. Run /graphify first.", file=sys.stderr) - sys.exit(1) - - if subcmd == "callflow-html": - from graphify.callflow_html import write_callflow_html as _write_callflow_html - out = _write_callflow_html( - graph=graph_path, - report=report_path, - labels=labels_path, - sections=sections_path, - output=callflow_output, - lang=callflow_lang, - max_sections=callflow_max_sections, - diagram_scale=callflow_diagram_scale, - max_diagram_nodes=callflow_max_diagram_nodes, - max_diagram_edges=callflow_max_diagram_edges, - verbose=True, - ) - print(f"callflow HTML written - open in any browser: {out}") - sys.exit(0) - - from networkx.readwrite import json_graph as _jg - from graphify.build import build_from_json as _bfj - from graphify.security import check_graph_file_size_cap as _check_cap - - # Solution 3 (#1019): for the HTML view, an oversized graph.json should - # not be a hard error. Detect the over-cap condition here and fall back - # to the community-aggregation view (node_limit=5000) below instead of - # exiting 1. All other subcommands keep the hard cap. - _over_cap = False - try: - _check_cap(graph_path) - except ValueError as _cap_err: - if subcmd == "html": - _over_cap = True - try: - _over_cap_bytes = graph_path.stat().st_size - except OSError: - _over_cap_bytes = -1 - print( - f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " - f"falling back to community-aggregation view (node_limit=5000)", - file=sys.stderr, - ) - else: - print(f"error: {_cap_err}", file=sys.stderr) - sys.exit(1) - _raw = json.loads(graph_path.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - try: - G = _jg.node_link_graph(_raw, edges="links") - except TypeError: - G = _jg.node_link_graph(_raw) - - # Load optional analysis/labels - communities: dict[int, list[str]] = {} - if analysis_path.exists(): - _an = json.loads(analysis_path.read_text(encoding="utf-8")) - communities = {int(k): v for k, v in _an.get("communities", {}).items()} - cohesion: dict[int, float] = {int(k): v for k, v in _an.get("cohesion", {}).items()} - gods_data = _an.get("gods", []) - else: - cohesion = {} - gods_data = [] - - # Fallback: graph.json carries the per-node community as a node attribute - # (`to_json` writes it on every node). The analysis sidecar is the - # canonical source — but the post-commit / watch rebuild path doesn't - # regenerate it, and `extract` may have its temp files cleaned up. When - # that happens, `graphify export html` previously bailed with - # "Single community - aggregated view not useful." even though the - # per-node attribute had the right data all along. Reconstruct from - # the graph itself so downstream subcommands (html, obsidian, wiki, - # svg, graphml, neo4j) don't silently produce a degraded artifact. - if not communities: - reconstructed: dict[int, list[str]] = {} - for node_id, data in G.nodes(data=True): - cid_raw = data.get("community") - if cid_raw is None: - continue - try: - cid = int(cid_raw) - except (TypeError, ValueError): - continue - reconstructed.setdefault(cid, []).append(str(node_id)) - if reconstructed: - communities = reconstructed - - labels: dict[int, str] = {} - if labels_path.exists(): - labels = {int(k): v for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items()} - - out_dir = graph_path.parent - - if subcmd == "html": - from graphify.export import to_html as _to_html - if no_viz: - html_target = out_dir / "graph.html" - if html_target.exists(): - html_target.unlink() - print("--no-viz: skipped graph.html") - else: - # Over-cap fallback (#1019): force the community-aggregation - # path so the oversized graph still renders a usable artifact. - _effective_node_limit = 5000 if _over_cap else node_limit - _to_html(G, communities, str(out_dir / "graph.html"), - community_labels=labels or None, node_limit=_effective_node_limit) - if G.number_of_nodes() <= _effective_node_limit: - print(f"graph.html written - open in any browser, no server needed") - if _over_cap: - sys.exit(0) - - elif subcmd == "obsidian": - from graphify.export import to_obsidian as _to_obsidian, to_canvas as _to_canvas - n = _to_obsidian(G, communities, str(obsidian_dir), - community_labels=labels or None, cohesion=cohesion or None) - print(f"Obsidian vault: {n} notes in {obsidian_dir}/") - _to_canvas(G, communities, str(obsidian_dir / "graph.canvas"), - community_labels=labels or None) - print(f"Canvas: {obsidian_dir}/graph.canvas") - print(f"Open {obsidian_dir}/ as a vault in Obsidian.") - - elif subcmd == "wiki": - from graphify.wiki import to_wiki as _to_wiki - from graphify.analyze import god_nodes as _god_nodes - if not communities: - print( - "error: .graphify_analysis.json is missing or empty — refusing to export wiki to prevent data loss.\n" - "Run `graphify extract .` (or `graphify cluster-only .`) to regenerate community data first.", - file=sys.stderr, - ) - sys.exit(1) - if not gods_data: - gods_data = _god_nodes(G) - n = _to_wiki(G, communities, str(out_dir / "wiki"), - community_labels=labels or None, cohesion=cohesion or None, - god_nodes_data=gods_data) - print(f"Wiki: {n} articles written to {out_dir}/wiki/") - print(f" {out_dir}/wiki/index.md -> agent entry point") - - elif subcmd == "svg": - from graphify.export import to_svg as _to_svg - _to_svg(G, communities, str(out_dir / "graph.svg"), - community_labels=labels or None) - print(f"graph.svg written - embeds in Obsidian, Notion, GitHub READMEs") - - elif subcmd == "graphml": - from graphify.export import to_graphml as _to_graphml - _to_graphml(G, communities, str(out_dir / "graph.graphml")) - print(f"graph.graphml written - open in Gephi, yEd, or any GraphML tool") - - elif subcmd == "neo4j": - if push_uri: - from graphify.export import push_to_neo4j as _push - if push_password is None: - print("error: --password required for --push", file=sys.stderr) - sys.exit(1) - result = _push(G, uri=push_uri, user=push_user, - password=push_password, communities=communities) - print(f"Pushed to Neo4j: {result['nodes']} nodes, {result['edges']} edges") - else: - from graphify.export import to_cypher as _to_cypher - _to_cypher(G, str(out_dir / "cypher.txt")) - print(f"cypher.txt written - import with: cypher-shell < {out_dir}/cypher.txt") - - elif subcmd == "falkordb": - if push_uri: - from graphify.export import push_to_falkordb as _push - result = _push(G, uri=push_uri, user=push_user, - password=push_password, communities=communities) - print(f"Pushed to FalkorDB: {result['nodes']} nodes, {result['edges']} edges") - else: - from graphify.export import to_cypher as _to_cypher - _to_cypher(G, str(out_dir / "cypher.txt")) - print(f"cypher.txt written ({out_dir}/cypher.txt) - statements are OpenCypher. " - f"FalkorDB's GRAPH.QUERY runs one statement at a time (no bulk script " - f"import), so load a graph with: graphify export falkordb --push " - f"falkordb://localhost:6379") - - elif cmd == "benchmark": - from graphify.benchmark import run_benchmark, print_benchmark - - graph_path = sys.argv[2] if len(sys.argv) > 2 else _default_graph_path() - _enforce_graph_size_cap_or_exit(Path(graph_path)) - # Try to load corpus_words from detect output - corpus_words = None - detect_path = Path(".graphify_detect.json") - if detect_path.exists(): - try: - detect_data = json.loads(detect_path.read_text(encoding="utf-8")) - corpus_words = detect_data.get("total_words") - except Exception: - pass - result = run_benchmark(graph_path, corpus_words=corpus_words) - print_benchmark(result) - - elif cmd == "global": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - from graphify.global_graph import ( - global_add as _global_add, - global_remove as _global_remove, - global_list as _global_list, - global_path as _global_path, - ) - if subcmd == "add": - # graphify global add [--as ] - args = sys.argv[3:] - source = None - tag = None - i = 0 - while i < len(args): - if args[i] == "--as" and i + 1 < len(args): - tag = args[i + 1]; i += 2 - elif not source: - source = Path(args[i]); i += 1 - else: - i += 1 - if not source: - print("Usage: graphify global add [--as ]", file=sys.stderr) - sys.exit(1) - tag = tag or source.parent.parent.name - try: - result = _global_add(source, tag) - if result["skipped"]: - print(f"'{tag}' unchanged since last add - global graph not modified.") - else: - print(f"Added '{tag}' to global graph: +{result['nodes_added']} nodes, " - f"-{result['nodes_removed']} pruned. Global: {_global_path()}") - except Exception as exc: - print(f"error: {exc}", file=sys.stderr); sys.exit(1) - elif subcmd == "remove": - tag = sys.argv[3] if len(sys.argv) > 3 else "" - if not tag: - print("Usage: graphify global remove ", file=sys.stderr); sys.exit(1) - try: - removed = _global_remove(tag) - print(f"Removed '{tag}' from global graph ({removed} nodes pruned).") - except KeyError as exc: - print(f"error: {exc}", file=sys.stderr); sys.exit(1) - elif subcmd == "list": - repos = _global_list() - if not repos: - print("Global graph is empty. Use 'graphify global add' to add a project.") - else: - print(f"Global graph: {_global_path()}") - for tag, info in repos.items(): - print(f" {tag}: {info.get('node_count', '?')} nodes, added {info.get('added_at', '?')[:10]}") - elif subcmd == "path": - print(_global_path()) - else: - print("Usage: graphify global [add|remove|list|path]", file=sys.stderr); sys.exit(1) - - elif cmd == "extract": - # Headless full-pipeline extraction for CI / scripts (#698). - # Runs detect -> AST extraction on code -> semantic LLM extraction on - # docs/papers/images -> merge -> build -> cluster -> write outputs. - # Unlike the skill.md path (which runs through Claude Code subagents), - # this calls extract_corpus_parallel directly using whichever backend - # has an API key set. - if len(sys.argv) < 3: - print( - "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " - "[--model M] [--mode deep] [--out DIR] [--google-workspace] [--no-cluster] " - "[--max-workers N] [--token-budget N] [--max-concurrency N] " - "[--api-timeout S] [--postgres DSN] [--cargo] [--timing]", - file=sys.stderr, - ) - sys.exit(1) - - has_path = True - if sys.argv[2].startswith("-"): - has_path = False - target = Path(".").resolve() - else: - target = Path(sys.argv[2]).resolve() - if not target.exists(): - print(f"error: path not found: {target}", file=sys.stderr) - sys.exit(1) - - backend: str | None = None - model: str | None = None - extract_mode: str | None = None - out_dir: Path | None = None - cli_postgres_dsn: str | None = None - cli_cargo: bool = False - no_cluster = False - dedup_llm = False - google_workspace = False - global_merge = False - code_only = False - global_repo_tag: str | None = None - # Performance/tuning knobs (issue #792). None means "use library default". - cli_max_workers: int | None = None - cli_token_budget: int | None = None - cli_max_concurrency: int | None = None - cli_api_timeout: float | None = None - # Clustering tuning knobs - cli_resolution: float = 1.0 - cli_exclude_hubs: float | None = None - cli_excludes: list[str] = [] - cli_timing: bool = False - - def _parse_int(name: str, raw: str) -> int: - try: - v = int(raw) - except ValueError: - print(f"error: {name} must be a positive integer (got {raw!r})", file=sys.stderr) - sys.exit(2) - if v <= 0: - print(f"error: {name} must be > 0 (got {v})", file=sys.stderr) - sys.exit(2) - return v - - def _parse_float(name: str, raw: str) -> float: - try: - v = float(raw) - except ValueError: - print(f"error: {name} must be a positive number (got {raw!r})", file=sys.stderr) - sys.exit(2) - if v <= 0: - print(f"error: {name} must be > 0 (got {v})", file=sys.stderr) - sys.exit(2) - return v - - args = sys.argv[3:] if has_path else sys.argv[2:] - i = 0 - while i < len(args): - a = args[i] - if a == "--backend" and i + 1 < len(args): - backend = args[i + 1]; i += 2 - elif a.startswith("--backend="): - backend = a.split("=", 1)[1]; i += 1 - elif a == "--model" and i + 1 < len(args): - model = args[i + 1]; i += 2 - elif a.startswith("--model="): - model = a.split("=", 1)[1]; i += 1 - elif a == "--mode" and i + 1 < len(args): - extract_mode = args[i + 1]; i += 2 - elif a.startswith("--mode="): - extract_mode = a.split("=", 1)[1]; i += 1 - elif a == "--out" and i + 1 < len(args): - out_dir = Path(args[i + 1]); i += 2 - elif a.startswith("--out="): - out_dir = Path(a.split("=", 1)[1]); i += 1 - elif a == "--no-cluster": - no_cluster = True; i += 1 - elif a == "--dedup-llm": - dedup_llm = True; i += 1 - elif a == "--code-only": - code_only = True; i += 1 - elif a == "--google-workspace": - google_workspace = True; i += 1 - elif a == "--global": - global_merge = True; i += 1 - elif a == "--as" and i + 1 < len(args): - global_repo_tag = args[i + 1]; i += 2 - elif a == "--max-workers" and i + 1 < len(args): - cli_max_workers = _parse_int("--max-workers", args[i + 1]); i += 2 - elif a.startswith("--max-workers="): - cli_max_workers = _parse_int("--max-workers", a.split("=", 1)[1]); i += 1 - elif a == "--token-budget" and i + 1 < len(args): - cli_token_budget = _parse_int("--token-budget", args[i + 1]); i += 2 - elif a.startswith("--token-budget="): - cli_token_budget = _parse_int("--token-budget", a.split("=", 1)[1]); i += 1 - elif a == "--max-concurrency" and i + 1 < len(args): - cli_max_concurrency = _parse_int("--max-concurrency", args[i + 1]); i += 2 - elif a.startswith("--max-concurrency="): - cli_max_concurrency = _parse_int("--max-concurrency", a.split("=", 1)[1]); i += 1 - elif a == "--api-timeout" and i + 1 < len(args): - cli_api_timeout = _parse_float("--api-timeout", args[i + 1]); i += 2 - elif a.startswith("--api-timeout="): - cli_api_timeout = _parse_float("--api-timeout", a.split("=", 1)[1]); i += 1 - elif a == "--resolution" and i + 1 < len(args): - cli_resolution = _parse_float("--resolution", args[i + 1]); i += 2 - elif a.startswith("--resolution="): - cli_resolution = _parse_float("--resolution", a.split("=", 1)[1]); i += 1 - elif a == "--exclude-hubs" and i + 1 < len(args): - cli_exclude_hubs = float(args[i + 1]); i += 2 - elif a.startswith("--exclude-hubs="): - cli_exclude_hubs = float(a.split("=", 1)[1]); i += 1 - elif a == "--exclude" and i + 1 < len(args): - cli_excludes.append(args[i + 1]); i += 2 - elif a.startswith("--exclude="): - cli_excludes.append(a.split("=", 1)[1]); i += 1 - elif a == "--postgres" and i + 1 < len(args): - cli_postgres_dsn = args[i + 1]; i += 2 - elif a.startswith("--postgres="): - cli_postgres_dsn = a.split("=", 1)[1]; i += 1 - elif a == "--cargo": - cli_cargo = True - i += 1 - elif a == "--timing": - cli_timing = True; i += 1 - else: - i += 1 - - if not has_path and cli_postgres_dsn is None: - print("error: must specify a path to scan or a --postgres DSN", file=sys.stderr) - sys.exit(1) - - _VALID_MODES = {"deep"} - if extract_mode is not None and extract_mode not in _VALID_MODES: - print( - f"error: unknown --mode '{extract_mode}'. " - f"Available: {', '.join(sorted(_VALID_MODES))}", - file=sys.stderr, - ) - sys.exit(2) - deep_mode = extract_mode == "deep" - if deep_mode: - print("[graphify extract] deep mode enabled: richer semantic extraction") - - # CLI flag wins over env var. Setting GRAPHIFY_API_TIMEOUT here so - # _call_openai_compat picks it up without needing a new kwarg path. - if cli_api_timeout is not None: - os.environ["GRAPHIFY_API_TIMEOUT"] = str(cli_api_timeout) - if cli_max_workers is not None: - os.environ["GRAPHIFY_MAX_WORKERS"] = str(cli_max_workers) - - # Resolve output dir. The user-facing contract is "/graphify-out/" - # so a fresh checkout writes graphify-out/ at the project root, matching - # the skill.md pipeline. - out_root = (out_dir.resolve() if out_dir else target) - graphify_out = out_root / _GRAPHIFY_OUT - graphify_out.mkdir(parents=True, exist_ok=True) - - stages = _StageTimer(cli_timing) - - from graphify.detect import ( - detect as _detect, - detect_incremental as _detect_incremental, - save_manifest as _save_manifest, - ) - manifest_path = graphify_out / "manifest.json" - existing_graph_path = graphify_out / "graph.json" - incremental_mode = manifest_path.exists() and existing_graph_path.exists() if has_path else False - - if not has_path: - code_files = [] - doc_files = [] - paper_files = [] - image_files = [] - deleted_files = [] - unchanged_total = 0 - files_by_type = {} - elif incremental_mode: - print(f"[graphify extract] incremental scan of {target}") - detection = _detect_incremental( - target, - manifest_path=str(manifest_path), - google_workspace=google_workspace or None, - extra_excludes=cli_excludes or None, - ) - files_by_type = detection.get("files", {}) - new_by_type = detection.get("new_files", {}) - code_files = [Path(p) for p in new_by_type.get("code", [])] - doc_files = [Path(p) for p in new_by_type.get("document", [])] - paper_files = [Path(p) for p in new_by_type.get("paper", [])] - image_files = [Path(p) for p in new_by_type.get("image", [])] - deleted_files = list(detection.get("deleted_files", [])) - unchanged_total = sum(len(v) for v in detection.get("unchanged_files", {}).values()) - else: - print(f"[graphify extract] scanning {target}") - detection = _detect(target, google_workspace=google_workspace or None, extra_excludes=cli_excludes or None) - files_by_type = detection.get("files", {}) - code_files = [Path(p) for p in files_by_type.get("code", [])] - doc_files = [Path(p) for p in files_by_type.get("document", [])] - paper_files = [Path(p) for p in files_by_type.get("paper", [])] - image_files = [Path(p) for p in files_by_type.get("image", [])] - deleted_files = [] - unchanged_total = 0 - - semantic_files = doc_files + paper_files + image_files - # --code-only: index code (pure local AST, no key) and skip the semantic - # (doc/paper/image) pass entirely, so a mixed repo doesn't hard-fail when no - # LLM backend is configured (#1734). Report what was skipped rather than - # silently dropping it. - if code_only and semantic_files: - print( - f"[graphify extract] --code-only: skipping {len(semantic_files)} " - f"non-code file(s) ({len(doc_files)} docs, {len(paper_files)} papers, " - f"{len(image_files)} images) — no LLM extraction" - ) - semantic_files = [] - doc_files = [] - paper_files = [] - image_files = [] - if incremental_mode: - print( - f"[graphify extract] {len(code_files)} code, {len(doc_files)} docs, " - f"{len(paper_files)} papers, {len(image_files)} images changed; " - f"{unchanged_total} unchanged; {len(deleted_files)} deleted" - ) - else: - print( - f"[graphify extract] found {len(code_files)} code, " - f"{len(doc_files)} docs, {len(paper_files)} papers, " - f"{len(image_files)} images" - ) - # Surface files that were seen but not classified (extensionless non-shebang - # project files like Dockerfile/Makefile, or unsupported extensions), so they - # are no longer invisible in graphify's own output (#1692). - _unclassified = detection.get("unclassified", []) if isinstance(detection, dict) else [] - if _unclassified: - _names = ", ".join(sorted({Path(p).name for p in _unclassified})[:6]) - _more = f" (+{len(_unclassified) - 6} more)" if len(_unclassified) > 6 else "" - print( - f"[graphify extract] {len(_unclassified)} file(s) not classified " - f"(no supported extension or shebang), skipped: {_names}{_more}" - ) - stages.mark("detect") - - # Resolve the LLM backend only now that we know whether the corpus - # needs one. A code-only corpus is pure local AST and must not require - # an API key; the key is enforced below only when there's LLM work. - from graphify.llm import ( - BACKENDS as _BACKENDS, - detect_backend as _detect_backend, - estimate_cost as _estimate_cost, - extract_corpus_parallel as _extract_corpus_parallel, - _format_backend_env_keys, - _get_backend_api_key, - ) - needs_llm = bool(semantic_files) or dedup_llm - if backend is None and needs_llm: - backend = _detect_backend() - if backend is not None and backend not in _BACKENDS: - print( - f"error: unknown backend '{backend}'. " - f"Available: {', '.join(sorted(_BACKENDS))}", - file=sys.stderr, - ) - sys.exit(1) - if needs_llm: - if backend is None: - reasons = [] - if semantic_files: - reasons.append( - f"{len(semantic_files)} doc/paper/image file(s) need semantic extraction" - ) - if dedup_llm: - reasons.append("--dedup-llm was passed") - hint = "" - if semantic_files: - hint = (" Or pass --code-only to index just the code " - "(local AST, no key) and skip the non-code files.") - print( - "error: no LLM API key found (" + "; ".join(reasons) + "). " - "Set GEMINI_API_KEY or GOOGLE_API_KEY (gemini), MOONSHOT_API_KEY " - "(kimi), ANTHROPIC_API_KEY (claude), OPENAI_API_KEY (openai), " - "DEEPSEEK_API_KEY (deepseek), or pass --backend. A code-only " - "corpus needs no key." + hint, - file=sys.stderr, - ) - sys.exit(1) - if backend == "ollama": - from graphify.llm import _validate_ollama_base_url - _oll_url = os.environ.get("OLLAMA_BASE_URL", _BACKENDS["ollama"].get("base_url", "")) - try: - _validate_ollama_base_url(_oll_url, warn=False) - except ValueError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(2) - if not _get_backend_api_key(backend): - allow_no_key = False - if backend == "ollama": - from urllib.parse import urlparse - ollama_url = os.environ.get( - "OLLAMA_BASE_URL", - _BACKENDS["ollama"].get("base_url", ""), - ) - try: - host = (urlparse(ollama_url).hostname or "").lower() - except Exception: - host = "" - allow_no_key = ( - host in ("localhost", "127.0.0.1", "::1") - or host.startswith("127.") - ) - elif backend == "bedrock": - allow_no_key = bool( - os.environ.get("AWS_PROFILE") - or os.environ.get("AWS_REGION") - or os.environ.get("AWS_DEFAULT_REGION") - or os.environ.get("AWS_ACCESS_KEY_ID") - ) - elif backend == "claude-cli": - import shutil as _shutil - allow_no_key = _shutil.which("claude") is not None - if not allow_no_key: - print( - "error: backend 'claude-cli' requires the `claude` CLI on $PATH " - "(install Claude Code and run `claude` once to authenticate).", - file=sys.stderr, - ) - sys.exit(1) - if not allow_no_key: - print( - f"error: backend '{backend}' requires {_format_backend_env_keys(backend)} to be set.", - file=sys.stderr, - ) - sys.exit(1) - - # AST extraction on code files. Empty code list (docs-only corpus) is - # the issue #698 case — skip cleanly instead of crashing inside extract(). - ast_result: dict = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} - if code_files: - from graphify.extract import extract as _ast_extract - # Anchor the cache at the output root, not the scanned project: - # with --out, a /graphify-out/cache/ would leak a - # graphify-out/ dir into a project that asked for external output. - ast_kwargs: dict = {"cache_root": out_root} - if cli_max_workers is not None: - ast_kwargs["max_workers"] = cli_max_workers - print(f"[graphify extract] AST extraction on {len(code_files)} code files...") - try: - ast_result = _ast_extract(code_files, **ast_kwargs) - except Exception as exc: - print(f"[graphify extract] AST extraction failed: {exc}", file=sys.stderr) - ast_result = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} - stages.mark("AST extract") - - # Semantic extraction on docs/papers/images. Check cache first. - from graphify.cache import ( - check_semantic_cache as _check_semantic_cache, - prune_semantic_cache as _prune_semantic_cache, - save_semantic_cache as _save_semantic_cache, - ) - sem_result: dict = { - "nodes": [], "edges": [], "hyperedges": [], - "input_tokens": 0, "output_tokens": 0, - } - sem_cache_hits = 0 - sem_cache_misses = 0 - if semantic_files: - sem_paths_str = [str(p) for p in semantic_files] - cached_nodes, cached_edges, cached_hyperedges, uncached_paths = ( - _check_semantic_cache(sem_paths_str, root=out_root) - ) - sem_cache_hits = len(semantic_files) - len(uncached_paths) - sem_cache_misses = len(uncached_paths) - sem_result["nodes"].extend(cached_nodes) - sem_result["edges"].extend(cached_edges) - sem_result["hyperedges"].extend(cached_hyperedges) - if sem_cache_hits: - print(f"[graphify extract] semantic cache: {sem_cache_hits} hit / {sem_cache_misses} miss") - - if uncached_paths: - print(f"[graphify extract] semantic extraction on {len(uncached_paths)} files via {backend}...") - corpus_kwargs: dict = { - "backend": backend, - "model": model, - "root": target, - } - if deep_mode: - corpus_kwargs["deep_mode"] = True - if cli_token_budget is not None: - corpus_kwargs["token_budget"] = cli_token_budget - if cli_max_concurrency is not None: - corpus_kwargs["max_concurrency"] = cli_max_concurrency - - # Minimal progress callback so the CLI is no longer silent - # during long local-inference runs (issue #792 addendum). - # Also track per-chunk success so we can fail loudly when - # every chunk errors (e.g. missing backend SDK package). - _chunk_stats = {"total": 0, "succeeded": 0} - def _progress(idx: int, total: int, _result: dict) -> None: - _chunk_stats["total"] = total - _chunk_stats["succeeded"] += 1 - print( - f"[graphify extract] chunk {idx + 1}/{total} done", - flush=True, - ) - corpus_kwargs["on_chunk_done"] = _progress - - try: - fresh = _extract_corpus_parallel( - [Path(p) for p in uncached_paths], - **corpus_kwargs, - ) - except ImportError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - except Exception as exc: - print( - f"[graphify extract] semantic extraction failed: {exc}", - file=sys.stderr, - ) - fresh = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - - # on_chunk_done only fires after a chunk succeeds. If fresh - # semantic extraction was requested and no chunks completed, - # fail instead of writing an AST-only graph with exit 0. - if uncached_paths and _chunk_stats["succeeded"] == 0: - print( - f"[graphify extract] error: all semantic chunks failed " - f"for backend '{backend}' ({len(uncached_paths)} uncached files) - " - f"see per-chunk errors above. If you see 'requires the X package', " - f"run `pip install X` and retry.", - file=sys.stderr, - ) - sys.exit(1) - try: - _save_semantic_cache( - fresh.get("nodes", []), - fresh.get("edges", []), - fresh.get("hyperedges", []), - root=out_root, - ) - except Exception as exc: - print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr) - sem_result["nodes"].extend(fresh.get("nodes", [])) - sem_result["edges"].extend(fresh.get("edges", [])) - sem_result["hyperedges"].extend(fresh.get("hyperedges", [])) - sem_result["input_tokens"] += fresh.get("input_tokens", 0) - sem_result["output_tokens"] += fresh.get("output_tokens", 0) - - # Prune orphaned semantic cache entries. The semantic cache is - # content-hash-keyed and unversioned, so it is never swept by the AST - # version-cleanup: every content change or file deletion leaves a - # permanent orphan that accumulates unbounded (#1527). Sweep it against - # the FULL live document set (``files_by_type`` — present in both the - # incremental and full branches), NOT the incremental ``semantic_files`` - # changed-subset, which would delete every unchanged doc's valid entry. - # Best-effort: a prune failure must never break extraction. - try: - from graphify.cache import file_hash as _file_hash - _live_hashes: set[str] = set() - for _kind in ("document", "paper", "image"): - for _fp in files_by_type.get(_kind, []): - _abs = Path(_fp) - if not _abs.is_absolute(): - _abs = Path(out_root) / _abs - if not _abs.is_file(): - continue # deleted/missing — leave out so its entry is pruned - try: - _live_hashes.add(_file_hash(_abs, out_root)) - except OSError: - pass - _prune_semantic_cache(out_root, _live_hashes) - except Exception as exc: - print(f"[graphify extract] warning: could not prune semantic cache: {exc}", file=sys.stderr) - stages.mark("semantic extract") - - pg_result: dict = {"nodes": [], "edges": []} - if cli_postgres_dsn is not None: - from graphify.pg_introspect import introspect_postgres - print(f"[graphify extract] introspecting PostgreSQL schema...") - try: - pg_result = introspect_postgres(cli_postgres_dsn) - except (ConnectionError, ImportError) as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - print(f"[graphify extract] PostgreSQL: {len(pg_result['nodes'])} nodes, " - f"{len(pg_result['edges'])} edges") - - cargo_result: dict = {"nodes": [], "edges": []} - if cli_cargo: - from graphify.cargo_introspect import introspect_cargo - print("[graphify extract] introspecting Cargo workspace...") - try: - cargo_result = introspect_cargo(target) - except (ConnectionError, ImportError, OSError) as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - print(f"[graphify extract] Cargo: {len(cargo_result['nodes'])} nodes, " - f"{len(cargo_result['edges'])} edges") - - # Merge AST + semantic + pg_result + cargo_result. Order matters for deduplication: passing AST - # first means semantic node attributes win on collision (richer labels - # for symbols also referenced in docs). Hyperedges only come from the - # semantic side. - merged: dict = { - "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])) + list(cargo_result.get("nodes", [])), - "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])) + list(cargo_result.get("edges", [])), - "hyperedges": list(sem_result.get("hyperedges", [])), - "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0), - "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0), - } - - graph_json_path = graphify_out / "graph.json" - analysis_path = graphify_out / ".graphify_analysis.json" - - # Build a manifest-safe files dict: only stamp semantic_hash for files - # that actually produced output (cache hit or fresh extraction). Files - # whose chunk failed have no source_file entry in sem_result — leaving - # their semantic_hash empty so detect_incremental re-queues them (#933). - _sem_extracted: set[str] = { - n.get("source_file", "") for n in sem_result.get("nodes", []) - } | { - e.get("source_file", "") for e in sem_result.get("edges", []) - } - _sem_extracted.discard("") - _sem_types = {"document", "paper", "image"} - _manifest_files = { - ftype: [f for f in flist if ftype not in _sem_types or f in _sem_extracted] - for ftype, flist in files_by_type.items() - } - - if no_cluster: - # --no-cluster: dump the raw merged extraction as graph.json. - # No NetworkX, no community detection, no analysis sidecar. - # Dedupe nodes (by id) and parallel edges so the raw output matches the - # clustered path (whose DiGraph collapses both) and stays deterministic - # across modes (#1317; node dedup also collapses shared Swift module - # anchors emitted per importing file, #1327). - from graphify.build import dedupe_edges as _dedupe_edges, dedupe_nodes as _dedupe_nodes - from graphify.export import backup_if_protected as _backup - if ( - incremental_mode - and not code_files - and not semantic_files - and not deleted_files - and not pg_result.get("nodes") - and not pg_result.get("edges") - and not cargo_result.get("nodes") - and not cargo_result.get("edges") - ): - print( - "[graphify extract] no incremental changes detected " - "(--no-cluster); outputs left untouched." - ) - try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) - stages.total() - sys.exit(0) - - merged["nodes"] = _dedupe_nodes(merged["nodes"]) - merged["edges"] = _dedupe_edges(merged["edges"]) - # Backfill source_file from endpoint nodes — this raw path bypasses - # build_from_json's backfill, and semantic edges sometimes omit it (#1279). - _node_sf = {n.get("id"): n.get("source_file") for n in merged["nodes"]} - for _e in merged["edges"]: - if not _e.get("source_file"): - _e["source_file"] = ( - _node_sf.get(_e.get("source")) or _node_sf.get(_e.get("target")) or "" - ) - _backup(graphify_out) - graph_json_path.write_text( - json.dumps(merged, indent=2), encoding="utf-8" - ) - stages.mark("write") - cost = _estimate_cost( - backend, merged["input_tokens"], merged["output_tokens"] - ) - print( - f"[graphify extract] wrote {graph_json_path} — " - f"{len(merged['nodes'])} nodes, {len(merged['edges'])} edges " - f"(no clustering)" - ) - if merged["input_tokens"] or merged["output_tokens"]: - print( - f"[graphify extract] tokens: " - f"{merged['input_tokens']:,} in / " - f"{merged['output_tokens']:,} out, " - f"est. cost: ${cost:.4f}" - ) - try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) - if global_merge: - from graphify.global_graph import global_add as _global_add - _tag = global_repo_tag or target.name - try: - result = _global_add(graphify_out / "graph.json", _tag) - if result["skipped"]: - print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") - else: - print(f"[graphify global] '{_tag}' merged into global graph " - f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") - except Exception as exc: - print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) - stages.total() - sys.exit(0) - - # Build graph + cluster + score + write. - from graphify.build import ( - build as _build, - build_from_json as _build_from_json, - build_merge as _build_merge, - ) - from graphify.cluster import cluster as _cluster, score_all as _score_all - from graphify.export import to_json as _to_json - from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising - dedup_backend = backend if dedup_llm else None - if incremental_mode: - G = _build_merge( - [merged], - graph_path=existing_graph_path, - prune_sources=deleted_files or None, - dedup=True, - dedup_llm_backend=dedup_backend, - root=target, - ) - else: - G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend, root=target) - stages.mark("build") - if G.number_of_nodes() == 0: - print( - "[graphify extract] graph is empty — extraction produced no nodes. " - "Possible causes: all files skipped, binary-only corpus, or LLM " - "returned no edges.", - file=sys.stderr, - ) - sys.exit(1) - - communities = _cluster(G, resolution=cli_resolution, exclude_hubs_percentile=cli_exclude_hubs) - stages.mark("cluster") - cohesion = _score_all(G, communities) - try: - gods = _god_nodes(G) - except Exception: - gods = [] - try: - surprises = _surprising(G, communities) - except Exception: - surprises = [] - stages.mark("analyze") - - from graphify.export import backup_if_protected as _backup - _backup(graphify_out) - _to_json(G, communities, str(graph_json_path), force=True) - stages.mark("export") - if merged.get("output_tokens", 0) > 0: - (graphify_out / ".graphify_semantic_marker").write_text( - json.dumps({"output_tokens": merged["output_tokens"]}), encoding="utf-8" - ) - if global_merge: - from graphify.global_graph import global_add as _global_add - _tag = global_repo_tag or target.name - try: - result = _global_add(graphify_out / "graph.json", _tag) - if result["skipped"]: - print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") - else: - print(f"[graphify global] '{_tag}' merged into global graph " - f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") - except Exception as exc: - print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) - analysis = { - "communities": {str(k): v for k, v in communities.items()}, - "cohesion": {str(k): v for k, v in cohesion.items()}, - "gods": gods, - "surprises": surprises, - "tokens": { - "input": merged["input_tokens"], - "output": merged["output_tokens"], - }, - } - analysis_path.write_text(json.dumps(analysis, indent=2), encoding="utf-8") - try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) - - cost = _estimate_cost(backend, merged["input_tokens"], merged["output_tokens"]) - print( - f"[graphify extract] wrote {graph_json_path}: " - f"{G.number_of_nodes()} nodes, {G.number_of_edges()} edges, " - f"{len(communities)} communities" - ) - print(f"[graphify extract] wrote {analysis_path}") - if incremental_mode: - print( - f"[graphify extract] incremental summary: " - f"{sem_cache_hits + unchanged_total} files cached/unchanged, " - f"{len(code_files) + sem_cache_misses} re-extracted, " - f"{len(deleted_files)} deleted" - ) - elif sem_cache_hits: - print(f"[graphify extract] semantic cache: {sem_cache_hits} cached, {sem_cache_misses} re-extracted") - if merged["input_tokens"] or merged["output_tokens"]: - print( - f"[graphify extract] tokens: " - f"{merged['input_tokens']:,} in / " - f"{merged['output_tokens']:,} out, " - f"est. cost (~{backend}): ${cost:.4f}" - ) - # extract intentionally stops at graph.json + analysis; the report and - # community labels are produced by `cluster-only` (or an agent's Step 5). - # Point standalone users at it so communities get named (#1097). - print( - "[graphify extract] next: run " - f"`graphify cluster-only {graphify_out.parent}` " - "to generate GRAPH_REPORT.md and name communities" - ) - stages.total() - - elif cmd == "cache-check": - # graphify cache-check [--root ] - # Reads file paths (one per line) from , checks semantic cache. - # Writes: - # graphify-out/.graphify_cached.json — already-cached nodes/edges/hyperedges - # graphify-out/.graphify_uncached.txt — paths that need extraction - # Stdout: "Cache: N hit, M miss" - from graphify.cache import check_semantic_cache - if len(sys.argv) < 3: - print("Usage: graphify cache-check [--root ]", file=sys.stderr) - sys.exit(1) - files_from = Path(sys.argv[2]) - root = Path(".") - i = 3 - while i < len(sys.argv): - if sys.argv[i] == "--root" and i + 1 < len(sys.argv): - root = Path(sys.argv[i + 1]) - i += 2 - else: - i += 1 - files = [f for f in files_from.read_text(encoding="utf-8").splitlines() if f.strip()] - cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(files, root) - out = root / _GRAPHIFY_OUT - out.mkdir(parents=True, exist_ok=True) - if cached_nodes or cached_edges or cached_hyperedges: - (out / ".graphify_cached.json").write_text( - json.dumps({"nodes": cached_nodes, "edges": cached_edges, "hyperedges": cached_hyperedges}, - ensure_ascii=False), - encoding="utf-8", - ) - (out / ".graphify_uncached.txt").write_text("\n".join(uncached), encoding="utf-8") - print(f"Cache: {len(files) - len(uncached)} hit, {len(uncached)} miss") - - elif cmd == "merge-chunks": - # graphify merge-chunks --out - # Concatenates .graphify_chunk_*.json files written by semantic subagents. - # Deduplicates nodes by id (first writer wins). Sums token counts. - import glob as _glob - if len(sys.argv) < 3: - print("Usage: graphify merge-chunks --out ", file=sys.stderr) - sys.exit(1) - out_path: Path | None = None - chunk_args: list[str] = [] - i = 2 - while i < len(sys.argv): - if sys.argv[i] == "--out" and i + 1 < len(sys.argv): - out_path = Path(sys.argv[i + 1]) - i += 2 - else: - chunk_args.append(sys.argv[i]) - i += 1 - if not out_path: - print("error: --out required", file=sys.stderr) - sys.exit(1) - chunk_files: list[str] = [] - for arg in chunk_args: - expanded = _glob.glob(arg) - chunk_files.extend(sorted(expanded) if expanded else [arg]) - merged: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - seen_ids: set[str] = set() - for cf in chunk_files: - try: - chunk = json.loads(Path(cf).read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: - print(f"[graphify merge-chunks] warning: skipping {cf}: {exc}", file=sys.stderr) - continue - for n in chunk.get("nodes", []): - if n.get("id") not in seen_ids: - seen_ids.add(n["id"]) - merged["nodes"].append(n) - merged["edges"].extend(chunk.get("edges", [])) - merged["hyperedges"].extend(chunk.get("hyperedges", [])) - merged["input_tokens"] += chunk.get("input_tokens", 0) - merged["output_tokens"] += chunk.get("output_tokens", 0) - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(merged, ensure_ascii=False), encoding="utf-8") - print( - f"Merged {len(chunk_files)} chunks: {len(merged['nodes'])} nodes, {len(merged['edges'])} edges, " - f"{merged['input_tokens']:,} in / {merged['output_tokens']:,} out tokens" - ) - - elif cmd == "merge-semantic": - # graphify merge-semantic --cached --new --out - # Merges cached semantic results with freshly-extracted chunk results. - # Deduplicates nodes by id (cached entries take priority over new ones). - if len(sys.argv) < 3: - print("Usage: graphify merge-semantic --cached --new --out ", file=sys.stderr) - sys.exit(1) - cached_path: Path | None = None - new_path: Path | None = None - out_path2: Path | None = None - i = 2 - while i < len(sys.argv): - if sys.argv[i] == "--cached" and i + 1 < len(sys.argv): - cached_path = Path(sys.argv[i + 1]); i += 2 - elif sys.argv[i] == "--new" and i + 1 < len(sys.argv): - new_path = Path(sys.argv[i + 1]); i += 2 - elif sys.argv[i] == "--out" and i + 1 < len(sys.argv): - out_path2 = Path(sys.argv[i + 1]); i += 2 - else: - i += 1 - if not out_path2: - print("error: --out required", file=sys.stderr) - sys.exit(1) - empty: dict = {"nodes": [], "edges": [], "hyperedges": []} - cached_data = json.loads(cached_path.read_text(encoding="utf-8")) if cached_path and cached_path.exists() else empty - new_data = json.loads(new_path.read_text(encoding="utf-8")) if new_path and new_path.exists() else empty - seen_ids2: set[str] = set() - all_nodes: list[dict] = [] - for n in cached_data.get("nodes", []) + new_data.get("nodes", []): - if n.get("id") not in seen_ids2: - seen_ids2.add(n["id"]) - all_nodes.append(n) - merged2 = { - "nodes": all_nodes, - "edges": cached_data.get("edges", []) + new_data.get("edges", []), - "hyperedges": cached_data.get("hyperedges", []) + new_data.get("hyperedges", []), - } - out_path2.parent.mkdir(parents=True, exist_ok=True) - out_path2.write_text(json.dumps(merged2, ensure_ascii=False), encoding="utf-8") - print(f"Merged: {len(merged2['nodes'])} nodes, {len(merged2['edges'])} edges") - - elif Path(cmd).exists() or cmd in (".", "..") or cmd.startswith(("./", "../", "/", "~")): - # User ran `graphify ` directly — treat as `graphify extract `. - # Common when following the PowerShell note in README (`graphify .`) or - # copy-pasting skill invocations without the leading slash. - sys.argv.insert(2, sys.argv[1]) - sys.argv[1] = "extract" - main() - else: - print(f"error: unknown command '{cmd}'", file=sys.stderr) - print("Run 'graphify --help' for usage.", file=sys.stderr) - sys.exit(1) + dispatch_command(cmd) if __name__ == "__main__": diff --git a/graphify/cli.py b/graphify/cli.py new file mode 100644 index 00000000..540a4351 --- /dev/null +++ b/graphify/cli.py @@ -0,0 +1,2753 @@ +"""graphify command dispatch — every non-install subcommand. + +Extracted verbatim from __main__.main(); __main__ now calls dispatch_command(cmd) +after the install/platform dispatch. Kept out of __main__ to shrink the CLI entry +module. The path-redirect (`graphify ` -> extract) re-enters via a lazy +import of main to avoid a cli<->__main__ import cycle. +""" +from __future__ import annotations +import json +import os +import sys +from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT +from pathlib import Path + + +_SEARCH_NUDGE = json.dumps({ + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "additionalContext": ( + 'MANDATORY: graphify-out/graph.json exists. You MUST run ' + '`graphify query ""` before grepping raw files. Only grep ' + 'after graphify has oriented you, or to modify/debug specific lines.' + ), + } +}, ensure_ascii=False, separators=(",", ":")) + "\n" +_READ_NUDGE = json.dumps({ + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "additionalContext": ( + 'MANDATORY: graphify-out/graph.json exists. You MUST run graphify ' + 'before reading source files. Use: `graphify query ""` ' + '(scoped subgraph), `graphify explain ""`, or ' + '`graphify path "" ""`. Only read raw files after graphify has ' + 'oriented you, or to modify/debug specific lines. This rule applies to ' + 'subagents too — include it in every subagent prompt involving code ' + 'exploration.' + ), + } +}, ensure_ascii=False, separators=(",", ":")) + "\n" +_HOOK_SOURCE_EXTS = ( + '.py', '.js', '.ts', '.tsx', '.jsx', '.astro', '.vue', '.svelte', '.go', + '.rs', '.java', '.rb', '.c', '.h', '.cpp', '.hpp', '.cc', '.cs', '.kt', + '.swift', '.php', '.scala', '.lua', '.sh', '.md', '.rst', '.txt', '.mdx', +) +_GEMINI_NUDGE_TEXT = ( + 'graphify: knowledge graph at graphify-out/. For focused questions, run ' + '`graphify query ""` (scoped subgraph, usually much smaller than ' + 'GRAPH_REPORT.md) instead of grepping raw files. Read GRAPH_REPORT.md only ' + 'for broad architecture context.' +) + + +def _default_graph_path() -> str: + return str(Path(_GRAPHIFY_OUT) / "graph.json") +class _StageTimer: + """Print per-stage wall-clock timings to stderr when --timing is set (#1490). + + Monotonic (perf_counter), diagnostic-only: emits ``[graphify timing] : + N.Ns`` after each stage and a final total. Off by default, so normal output is + byte-identical and machine-read stdout is untouched. + """ + + def __init__(self, enabled: bool) -> None: + import time as _time + self._now = _time.perf_counter + self.enabled = enabled + self.start = self._now() + self._last = self.start + + def mark(self, stage: str) -> None: + now = self._now() + if self.enabled: + print(f"[graphify timing] {stage}: {now - self._last:.1f}s", file=sys.stderr) + self._last = now + + def total(self) -> None: + if self.enabled: + print(f"[graphify timing] total: {self._now() - self.start:.1f}s", file=sys.stderr) +def _enforce_graph_size_cap_or_exit(gp: Path) -> None: + """Reject oversized graph files before parsing (CLI exit-on-fail flavor). + + Delegates to ``graphify.security.check_graph_file_size_cap`` and turns the + raised ``ValueError`` into a CLI-style ``error: ...`` message + exit 1. + Use this from ``__main__.py`` subcommands that already use the ``print + + sys.exit(1)`` idiom. Library/MCP/loader callers (``serve._load_graph``, + ``build``, ``benchmark``, ``tree_html``, ``callflow_html``, ``prs``, + ``global_graph``, ``watch``, ``export``) call the security helper directly + and let the ``ValueError`` propagate. + """ + from graphify.security import check_graph_file_size_cap + try: + check_graph_file_size_cap(gp) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) +def _run_hook_guard(kind: str) -> None: + """Shell-agnostic PreToolUse guard (#522). + + Reads the tool-call JSON from stdin and, when a knowledge graph exists in the + current output dir, prints a nudge (`additionalContext`) telling the agent to + use graphify instead of grepping/reading raw files. Replaces the old inline + bash hooks that failed to parse on Windows. Always fails open: any error, or a + non-matching tool call, prints nothing and the caller exits 0, so a legitimate + tool call is never blocked. Detection mirrors the previous hooks exactly. + """ + from graphify.paths import out_path, GRAPHIFY_OUT_NAME + # Gemini's BeforeTool hook takes no stdin and must ALWAYS return a decision so + # the tool is never blocked; the graph nudge is appended only when a graph + # exists. Handled before the stdin read below (which the search/read guards need). + if kind == "gemini": + payload = {"decision": "allow"} + try: + if out_path("graph.json").is_file(): + payload["additionalContext"] = _GEMINI_NUDGE_TEXT + except Exception: + pass + sys.stdout.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":"))) + return + try: + d = json.loads(sys.stdin.buffer.read().decode("utf-8", "replace")) + except Exception: + return + if not isinstance(d, dict): + return + t = d.get("tool_input", d) + if not isinstance(t, dict): + return + try: + if kind == "search": + cmd_str = str(t.get("command", "") or "") + # Same set the old `case` matched: *grep*, *ripgrep*, and rg/find/fd/ + # ack/ag as a token (name followed by a space). + if any(tok in cmd_str for tok in ("grep", "ripgrep", "rg ", "find ", "fd ", "ack ", "ag ")) \ + and out_path("graph.json").is_file(): + sys.stdout.write(_SEARCH_NUDGE) + elif kind == "read": + vals = [str(t.get("file_path") or ""), str(t.get("pattern") or ""), str(t.get("path") or "")] + j = " ".join(vals).lower().replace("\\", "/") + tails = [ + "." + seg.rsplit(".", 1)[-1] + for v in vals if v + for seg in [v.lower().replace("\\", "/").rsplit("/", 1)[-1]] + if "." in seg + ] + under_out = "graphify-out/" in j or (GRAPHIFY_OUT_NAME.lower() + "/") in j + if not under_out and any(tl in _HOOK_SOURCE_EXTS for tl in tails) \ + and out_path("graph.json").is_file(): + sys.stdout.write(_READ_NUDGE) + except Exception: + pass +def _clone_repo( + url: str, branch: str | None = None, out_dir: Path | None = None +) -> Path: + """Clone a GitHub repo to a local cache dir and return the path. + + Clones into ~/.graphify/repos// by default so repeated + runs on the same URL reuse the existing clone (git pull instead of clone). + """ + import subprocess as _sp + import re as _re + + # Normalise URL — strip trailing .git if present + url = url.rstrip("/") + if not url.endswith(".git"): + git_url = url + ".git" + else: + git_url = url + url = url[:-4] + + # Extract owner/repo from URL + m = _re.search(r"github\.com[:/]([^/]+)/([^/]+?)(?:\.git)?$", url) + if not m: + print(f"error: not a recognised GitHub URL: {url}", file=sys.stderr) + sys.exit(1) + owner, repo = m.group(1), m.group(2) + + if out_dir: + dest = out_dir + else: + dest = Path.home() / ".graphify" / "repos" / owner / repo + + if branch and branch.startswith("-"): + print(f"error: invalid branch name: {branch!r}", file=sys.stderr) + sys.exit(1) + + if dest.exists(): + print(f"Repo already cloned at {dest} - pulling latest...", flush=True) + cmd = ["git", "-C", str(dest), "pull"] + if branch: + cmd += ["origin", "--", branch] + result = _sp.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"warning: git pull failed:\n{result.stderr}", file=sys.stderr) + else: + dest.parent.mkdir(parents=True, exist_ok=True) + print(f"Cloning {url} -> {dest} ...", flush=True) + cmd = ["git", "clone", "--depth", "1"] + if branch: + cmd += ["--branch", branch] + cmd += ["--", git_url, str(dest)] + result = _sp.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"error: git clone failed:\n{result.stderr}", file=sys.stderr) + sys.exit(1) + + print(f"Ready at: {dest}", flush=True) + return dest + + +def _reenter_main() -> None: + from graphify.__main__ import main + main() + + +def dispatch_command(cmd: str) -> None: + if cmd == "provider": + from graphify.llm import _custom_providers_path, BACKENDS + import json as _json + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + global_path = _custom_providers_path(global_=True) + + if subcmd == "list": + global_path.parent.mkdir(parents=True, exist_ok=True) + existing: dict = {} + if global_path.is_file(): + try: + existing = _json.loads(global_path.read_text(encoding="utf-8")) + except Exception: + pass + if not existing: + print("No custom providers registered.") + else: + for name in existing: + print(f" {name} ({existing[name].get('base_url', '')})") + + elif subcmd == "show": + name = sys.argv[3] if len(sys.argv) > 3 else "" + if not name: + print("Usage: graphify provider show ", file=sys.stderr) + sys.exit(1) + existing = {} + if global_path.is_file(): + try: + existing = _json.loads(global_path.read_text(encoding="utf-8")) + except Exception: + pass + if name not in existing: + print(f"Provider '{name}' not found.", file=sys.stderr) + sys.exit(1) + print(_json.dumps({name: existing[name]}, indent=2)) + + elif subcmd == "add": + args = sys.argv[3:] + name = args[0] if args and not args[0].startswith("-") else "" + if not name: + print("Usage: graphify provider add --base-url URL --default-model MODEL --env-key KEY", file=sys.stderr) + sys.exit(1) + if name in BACKENDS: + print(f"Error: '{name}' is a built-in provider and cannot be overridden.", file=sys.stderr) + sys.exit(1) + base_url = "" + default_model = "" + env_key = "" + pricing_input = 0.0 + pricing_output = 0.0 + i = 1 + while i < len(args): + a = args[i] + if a == "--base-url" and i + 1 < len(args): + base_url = args[i + 1]; i += 2 + elif a.startswith("--base-url="): + base_url = a.split("=", 1)[1]; i += 1 + elif a == "--default-model" and i + 1 < len(args): + default_model = args[i + 1]; i += 2 + elif a.startswith("--default-model="): + default_model = a.split("=", 1)[1]; i += 1 + elif a == "--env-key" and i + 1 < len(args): + env_key = args[i + 1]; i += 2 + elif a.startswith("--env-key="): + env_key = a.split("=", 1)[1]; i += 1 + elif a == "--pricing-input" and i + 1 < len(args): + pricing_input = float(args[i + 1]); i += 2 + elif a == "--pricing-output" and i + 1 < len(args): + pricing_output = float(args[i + 1]); i += 2 + else: + i += 1 + if not base_url or not default_model or not env_key: + print("Error: --base-url, --default-model, and --env-key are required.", file=sys.stderr) + sys.exit(1) + from graphify.llm import provider_base_url_ok + if not provider_base_url_ok(base_url, name): + print(f"Error: refusing to add provider with unsafe base_url {base_url!r}.", file=sys.stderr) + sys.exit(1) + global_path.parent.mkdir(parents=True, exist_ok=True) + existing = {} + if global_path.is_file(): + try: + existing = _json.loads(global_path.read_text(encoding="utf-8")) + except Exception: + pass + existing[name] = { + "base_url": base_url, + "default_model": default_model, + "env_key": env_key, + "pricing": {"input": pricing_input, "output": pricing_output}, + "temperature": 0, + } + global_path.write_text(_json.dumps(existing, indent=2) + "\n", encoding="utf-8") + print(f"Provider '{name}' added. Use with: graphify extract . --backend {name}") + + elif subcmd == "remove": + name = sys.argv[3] if len(sys.argv) > 3 else "" + if not name: + print("Usage: graphify provider remove ", file=sys.stderr) + sys.exit(1) + existing = {} + if global_path.is_file(): + try: + existing = _json.loads(global_path.read_text(encoding="utf-8")) + except Exception: + pass + if name not in existing: + print(f"Provider '{name}' not found.", file=sys.stderr) + sys.exit(1) + del existing[name] + global_path.write_text(_json.dumps(existing, indent=2) + "\n", encoding="utf-8") + print(f"Provider '{name}' removed.") + + else: + print("Usage: graphify provider [add|list|show|remove]", file=sys.stderr) + if subcmd: + sys.exit(1) + elif cmd == "prs": + from graphify.prs import cmd_prs + cmd_prs(sys.argv[2:]) + elif cmd == "hook": + from graphify.hooks import ( + install as hook_install, + uninstall as hook_uninstall, + status as hook_status, + ) + + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd == "install": + print(hook_install(Path("."))) + elif subcmd == "uninstall": + print(hook_uninstall(Path("."))) + elif subcmd == "status": + print(hook_status(Path("."))) + else: + print("Usage: graphify hook [install|uninstall|status]", file=sys.stderr) + sys.exit(1) + elif cmd == "query": + if len(sys.argv) < 3: + print("Usage: graphify query \"\" [--dfs] [--context C] [--budget N] [--graph path]", file=sys.stderr) + sys.exit(1) + from graphify.serve import _query_graph_text + from graphify.security import sanitize_label + from networkx.readwrite import json_graph + from graphify import querylog + + question = sys.argv[2] + use_dfs = "--dfs" in sys.argv + budget = 2000 + graph_path = _default_graph_path() + context_filters: list[str] = [] + args = sys.argv[3:] + i = 0 + while i < len(args): + if args[i] == "--budget" and i + 1 < len(args): + try: + budget = int(args[i + 1]) + except ValueError: + print(f"error: --budget must be an integer", file=sys.stderr) + sys.exit(1) + i += 2 + elif args[i].startswith("--budget="): + try: + budget = int(args[i].split("=", 1)[1]) + except ValueError: + print(f"error: --budget must be an integer", file=sys.stderr) + sys.exit(1) + i += 1 + elif args[i] == "--context" and i + 1 < len(args): + context_filters.append(args[i + 1]) + i += 2 + elif args[i].startswith("--context="): + context_filters.append(args[i].split("=", 1)[1]) + i += 1 + elif args[i] == "--graph" and i + 1 < len(args): + graph_path = args[i + 1] + i += 2 + else: + i += 1 + gp = Path(graph_path).resolve() + if not gp.exists(): + print(f"error: graph file not found: {gp}", file=sys.stderr) + sys.exit(1) + if not gp.suffix == ".json": + print(f"error: graph file must be a .json file", file=sys.stderr) + sys.exit(1) + _enforce_graph_size_cap_or_exit(gp) + try: + import json as _json + import networkx as _nx + + _raw = _json.loads(gp.read_text(encoding="utf-8")) + if "links" not in _raw and "edges" in _raw: + _raw = dict(_raw, links=_raw["edges"]) + try: + G = json_graph.node_link_graph(_raw, edges="links") + except TypeError: + G = json_graph.node_link_graph(_raw) + try: + from graphify.build import graph_has_legacy_ids as _legacy + if _legacy(_raw.get("nodes", [])): + print( + "[graphify] note: this graph uses the pre-#1504 node-ID scheme; " + "rebuild with `graphify extract --force` to get path-qualified IDs " + "(fixes same-name-file collisions).", + file=sys.stderr, + ) + except Exception: + pass + except Exception as exc: + print(f"error: could not load graph: {exc}", file=sys.stderr) + sys.exit(1) + import time as _time + _t0 = _time.perf_counter() + _mode = "dfs" if use_dfs else "bfs" + _result = _query_graph_text( + G, + question, + mode=_mode, + depth=2, + token_budget=budget, + context_filters=context_filters, + ) + querylog.log_query( + kind="query", + question=question, + corpus=str(gp), + result=_result, + mode=_mode, + depth=2, + token_budget=budget, + duration_ms=(_time.perf_counter() - _t0) * 1000, + ) + print(_result) + elif cmd == "affected": + if len(sys.argv) < 3: + print("Usage: graphify affected \"\" [--relation R] [--depth N] [--graph path]", file=sys.stderr) + sys.exit(1) + from graphify.affected import DEFAULT_AFFECTED_RELATIONS, format_affected, load_graph + query = sys.argv[2] + graph_path = _default_graph_path() + depth = 2 + relations: list[str] = [] + args = sys.argv[3:] + i = 0 + while i < len(args): + if args[i] == "--graph" and i + 1 < len(args): + graph_path = args[i + 1] + i += 2 + elif args[i].startswith("--graph="): + graph_path = args[i].split("=", 1)[1] + i += 1 + elif args[i] == "--depth" and i + 1 < len(args): + try: + depth = int(args[i + 1]) + except ValueError: + print("error: --depth must be an integer", file=sys.stderr) + sys.exit(1) + i += 2 + elif args[i].startswith("--depth="): + try: + depth = int(args[i].split("=", 1)[1]) + except ValueError: + print("error: --depth must be an integer", file=sys.stderr) + sys.exit(1) + i += 1 + elif args[i] == "--relation" and i + 1 < len(args): + relations.append(args[i + 1]) + i += 2 + elif args[i].startswith("--relation="): + relations.append(args[i].split("=", 1)[1]) + i += 1 + else: + i += 1 + gp = Path(graph_path).resolve() + if not gp.exists(): + print(f"error: graph file not found: {gp}", file=sys.stderr) + sys.exit(1) + if not gp.suffix == ".json": + print("error: graph file must be a .json file", file=sys.stderr) + sys.exit(1) + try: + graph = load_graph(gp) + except Exception as exc: + print(f"error: could not load graph: {exc}", file=sys.stderr) + sys.exit(1) + print( + format_affected( + graph, + query, + relations=relations or DEFAULT_AFFECTED_RELATIONS, + depth=depth, + ) + ) + elif cmd == "save-result": + # graphify save-result --question Q --answer A [--type T] [--nodes N1 N2 ...] + # [--outcome useful|dead_end|corrected] [--correction TEXT] + import argparse as _ap + + p = _ap.ArgumentParser(prog="graphify save-result") + p.add_argument("--question", required=True) + p.add_argument("--answer", default=None) + p.add_argument("--answer-file", dest="answer_file", default=None) + p.add_argument("--type", dest="query_type", default="query") + p.add_argument("--nodes", nargs="*", default=[]) + p.add_argument("--outcome", choices=("useful", "dead_end", "corrected"), default=None) + p.add_argument("--correction", default=None) + p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) + opts = p.parse_args(sys.argv[2:]) + if opts.answer_file: + opts.answer = Path(opts.answer_file).read_text(encoding="utf-8").strip() + elif not opts.answer: + p.error("--answer or --answer-file is required") + from graphify.ingest import save_query_result as _sqr + + out = _sqr( + question=opts.question, + answer=opts.answer, + memory_dir=Path(opts.memory_dir), + query_type=opts.query_type, + source_nodes=opts.nodes or None, + outcome=opts.outcome, + correction=opts.correction, + ) + print(f"Saved to {out}") + elif cmd == "reflect": + import argparse as _ap + + p = _ap.ArgumentParser(prog="graphify reflect") + p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) + p.add_argument( + "--out", + default=str(Path(_GRAPHIFY_OUT) / "reflections" / "LESSONS.md"), + ) + p.add_argument("--graph", default=None) + p.add_argument("--analysis", default=None) + p.add_argument("--labels", default=None) + p.add_argument("--half-life-days", type=float, default=30.0, + help="signal weight halves every N days (default 30)") + p.add_argument("--min-corroboration", type=int, default=2, + help="distinct useful results to promote a node to preferred (default 2)") + p.add_argument("--if-stale", action="store_true", + help="skip when LESSONS.md is already newer than every input " + "(e.g. the git hook just refreshed it)") + opts = p.parse_args(sys.argv[2:]) + from graphify.reflect import reflect as _reflect, lessons_fresh as _lessons_fresh + + graph_arg = opts.graph + if graph_arg is None: + default_graph = Path(_GRAPHIFY_OUT) / "graph.json" + if default_graph.exists(): + graph_arg = str(default_graph) + + _gp = Path(graph_arg) if graph_arg else None + _analysis_path = None + _labels_path = None + if _gp is not None: + _analysis_path = Path(opts.analysis) if opts.analysis else ( + _gp.parent / ".graphify_analysis.json") + _labels_path = Path(opts.labels) if opts.labels else ( + _gp.parent / ".graphify_labels.json") + + if opts.if_stale and _lessons_fresh( + Path(opts.out), Path(opts.memory_dir), _gp, _analysis_path, _labels_path + ): + print(f"Lessons already up to date -> {opts.out} (skipped; omit --if-stale to force)") + else: + out_path, agg = _reflect( + memory_dir=Path(opts.memory_dir), + out_path=Path(opts.out), + graph_path=_gp, + analysis_path=_analysis_path, + labels_path=_labels_path, + half_life_days=opts.half_life_days, + min_corroboration=opts.min_corroboration, + ) + c = agg["counts"] + print( + f"Reflected {agg['total']} memories " + f"({c['useful']} useful, {c['dead_end']} dead ends, " + f"{c['corrected']} corrected) -> {out_path}" + ) + elif cmd == "path": + if len(sys.argv) < 4: + print( + 'Usage: graphify path "" "" [--graph path]', + file=sys.stderr, + ) + sys.exit(1) + from graphify.serve import _score_nodes + from networkx.readwrite import json_graph + import networkx as _nx + + source_label = sys.argv[2] + target_label = sys.argv[3] + graph_path = _default_graph_path() + args = sys.argv[4:] + for i, a in enumerate(args): + if a == "--graph" and i + 1 < len(args): + graph_path = args[i + 1] + gp = Path(graph_path).resolve() + if not gp.exists(): + print(f"error: graph file not found: {gp}", file=sys.stderr) + sys.exit(1) + _enforce_graph_size_cap_or_exit(gp) + _raw = json.loads(gp.read_text(encoding="utf-8")) + if "links" not in _raw and "edges" in _raw: + _raw = dict(_raw, links=_raw["edges"]) + # Force directed so the renderer can recover stored caller→callee direction. + _raw = {**_raw, "directed": True} + try: + G = json_graph.node_link_graph(_raw, edges="links") + except TypeError: + G = json_graph.node_link_graph(_raw) + src_scored = _score_nodes(G, [t.lower() for t in source_label.split()]) + tgt_scored = _score_nodes(G, [t.lower() for t in target_label.split()]) + if not src_scored: + print(f"No node matching '{source_label}' found.", file=sys.stderr) + sys.exit(1) + if not tgt_scored: + print(f"No node matching '{target_label}' found.", file=sys.stderr) + sys.exit(1) + src_nid, tgt_nid = src_scored[0][1], tgt_scored[0][1] + # Ambiguity guard: when both queries resolve to the same node, the + # shortest path is trivially zero hops, which is almost never what the + # caller wanted (see bug #828). + if src_nid == tgt_nid: + print( + f"'{source_label}' and '{target_label}' both resolved to the same " + f"node '{src_nid}'. Use a more specific label or the exact node ID.", + file=sys.stderr, + ) + sys.exit(1) + for _name, _scored in (("source", src_scored), ("target", tgt_scored)): + if len(_scored) >= 2: + _top, _runner = _scored[0][0], _scored[1][0] + if _top > 0 and (_top - _runner) / _top < 0.10: + print( + f"warning: {_name} match was ambiguous " + f"(top score {_top:g}, runner-up {_runner:g})", + file=sys.stderr, + ) + try: + path_nodes = _nx.shortest_path(G.to_undirected(as_view=True), src_nid, tgt_nid) + except (_nx.NetworkXNoPath, _nx.NodeNotFound): + print(f"No path found between '{source_label}' and '{target_label}'.") + sys.exit(0) + hops = len(path_nodes) - 1 + segments = [] + from graphify.build import edge_data + for i in range(len(path_nodes) - 1): + u, v = path_nodes[i], path_nodes[i + 1] + # Check which direction the stored edge points. + if G.has_edge(u, v): + edata = edge_data(G, u, v) + forward = True + else: + edata = edge_data(G, v, u) + forward = False + rel = edata.get("relation", "") + conf = edata.get("confidence", "") + conf_str = f" [{conf}]" if conf else "" + if i == 0: + segments.append(G.nodes[u].get("label", u)) + if forward: + segments.append(f"--{rel}{conf_str}--> {G.nodes[v].get('label', v)}") + else: + segments.append(f"<--{rel}{conf_str}-- {G.nodes[v].get('label', v)}") + print(f"Shortest path ({hops} hops):\n " + " ".join(segments)) + from graphify import querylog + querylog.log_query( + kind="path", + question=f"{sys.argv[2]} -> {sys.argv[3]}", + corpus=str(gp), + nodes_returned=hops, + ) + + elif cmd == "explain": + if len(sys.argv) < 3: + print('Usage: graphify explain "" [--graph path]', file=sys.stderr) + sys.exit(1) + from graphify.serve import _find_node + from networkx.readwrite import json_graph + + label = sys.argv[2] + graph_path = _default_graph_path() + args = sys.argv[3:] + for i, a in enumerate(args): + if a == "--graph" and i + 1 < len(args): + graph_path = args[i + 1] + gp = Path(graph_path).resolve() + if not gp.exists(): + print(f"error: graph file not found: {gp}", file=sys.stderr) + sys.exit(1) + _enforce_graph_size_cap_or_exit(gp) + _raw = json.loads(gp.read_text(encoding="utf-8")) + if "links" not in _raw and "edges" in _raw: + _raw = dict(_raw, links=_raw["edges"]) + # Force directed so the renderer can recover stored caller→callee direction. + _raw = {**_raw, "directed": True} + try: + G = json_graph.node_link_graph(_raw, edges="links") + except TypeError: + G = json_graph.node_link_graph(_raw) + matches = _find_node(G, label) + if not matches: + print(f"No node matching '{label}' found.") + sys.exit(0) + nid = matches[0] + d = G.nodes[nid] + print(f"Node: {d.get('label', nid)}") + print(f" ID: {nid}") + print( + f" Source: {d.get('source_file', '')} {d.get('source_location', '')}".rstrip() + ) + print(f" Type: {d.get('file_type', '')}") + print(f" Community: {d.get('community_name') or d.get('community', '')}") + # Work-memory overlay: a derived experiential hint from `graphify reflect`, + # merged in display-only from the .graphify_learning.json sidecar next to + # graph.json. No line when the node has no overlay entry. + try: + from graphify.reflect import load_learning_overlay as _llo + from graphify.security import sanitize_label as _sl + _overlay = _llo(gp) + _entry = _overlay.get(str(nid)) + if _entry: + _status = _sl(str(_entry.get("status", ""))) + if _status == "contested": + _line = (f" Lesson: contested (useful {_entry.get('uses', 0)} / " + f"dead-end {_entry.get('neg', 0)})") + elif _status == "preferred": + _line = (f" Lesson: preferred source (start here) — " + f"{_entry.get('uses', 0)} useful, score={_entry.get('score', 0)}") + else: + _line = (f" Lesson: {_status or 'tentative'} — " + f"{_entry.get('uses', 0)} useful, score={_entry.get('score', 0)}") + if _entry.get("stale"): + _line += " [code changed since — re-verify]" + print(_line) + except Exception: + pass + print(f" Degree: {G.degree(nid)}") + from graphify.build import edge_data + connections: list[tuple[str, str, dict]] = [] # (direction, neighbor_id, edge_data) + for nb in G.successors(nid): + connections.append(("out", nb, edge_data(G, nid, nb))) + for nb in G.predecessors(nid): + connections.append(("in", nb, edge_data(G, nb, nid))) + if connections: + print(f"\nConnections ({len(connections)}):") + connections.sort(key=lambda c: G.degree(c[1]), reverse=True) + for direction, nb, edata in connections[:20]: + rel = edata.get("relation", "") + conf = edata.get("confidence", "") + arrow = "-->" if direction == "out" else "<--" + print(f" {arrow} {G.nodes[nb].get('label', nb)} [{rel}] [{conf}]") + if len(connections) > 20: + print(f" ... and {len(connections) - 20} more") + from graphify import querylog + querylog.log_query( + kind="explain", + question=sys.argv[2], + corpus=str(gp), + nodes_returned=len(connections), + ) + + elif cmd == "diagnose": + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd != "multigraph": + print( + "Usage: graphify diagnose multigraph " + "[--graph path] [--json] [--max-examples N] " + "[--directed] [--undirected] [--extract-path path]", + file=sys.stderr, + ) + sys.exit(1) + + graph_path = Path(_default_graph_path()) + max_examples = 5 + directed: bool | None = None + direction_flag: str | None = None + json_output = False + extract_path: Path | None = None + + i = 3 + while i < len(sys.argv): + arg = sys.argv[i] + if arg == "--graph": + i += 1 + if i >= len(sys.argv): + print("error: --graph requires a path", file=sys.stderr) + sys.exit(1) + graph_path = Path(sys.argv[i]) + elif arg == "--json": + json_output = True + elif arg == "--max-examples": + i += 1 + if i >= len(sys.argv): + print("error: --max-examples requires an integer", file=sys.stderr) + sys.exit(1) + try: + max_examples = int(sys.argv[i]) + except ValueError: + print("error: --max-examples requires an integer", file=sys.stderr) + sys.exit(1) + if max_examples < 0: + print("error: --max-examples must be >= 0", file=sys.stderr) + sys.exit(1) + elif arg == "--directed": + if direction_flag == "undirected": + print( + "error: --directed and --undirected are mutually exclusive", + file=sys.stderr, + ) + sys.exit(1) + direction_flag = "directed" + directed = True + elif arg == "--undirected": + if direction_flag == "directed": + print( + "error: --directed and --undirected are mutually exclusive", + file=sys.stderr, + ) + sys.exit(1) + direction_flag = "undirected" + directed = False + elif arg == "--extract-path": + i += 1 + if i >= len(sys.argv): + print("error: --extract-path requires a path", file=sys.stderr) + sys.exit(1) + extract_path = Path(sys.argv[i]) + else: + print(f"error: unknown diagnose option {arg}", file=sys.stderr) + sys.exit(1) + i += 1 + + from graphify.diagnostics import ( + diagnose_file, + format_diagnostic_json, + format_diagnostic_report, + ) + + try: + summary = diagnose_file( + graph_path, + directed=directed, + root=Path(".").resolve(), + max_examples=max_examples, + extract_path=extract_path, + ) + except Exception as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + + if json_output: + print(json.dumps(format_diagnostic_json(summary), indent=2)) + else: + print(format_diagnostic_report(summary)) + + elif cmd == "add": + if len(sys.argv) < 3: + print( + "Usage: graphify add [--author Name] [--contributor Name] [--dir ./raw]", + file=sys.stderr, + ) + sys.exit(1) + from graphify.ingest import ingest as _ingest + + url = sys.argv[2] + author: str | None = None + contributor: str | None = None + target_dir = Path("raw") + args = sys.argv[3:] + i = 0 + while i < len(args): + if args[i] == "--author" and i + 1 < len(args): + author = args[i + 1] + i += 2 + elif args[i] == "--contributor" and i + 1 < len(args): + contributor = args[i + 1] + i += 2 + elif args[i] == "--dir" and i + 1 < len(args): + target_dir = Path(args[i + 1]) + i += 2 + else: + i += 1 + try: + saved = _ingest(url, target_dir, author=author, contributor=contributor) + print(f"Saved to {saved}") + print("Run /graphify --update in your AI assistant to update the graph.") + except Exception as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + + elif cmd == "watch": + watch_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(".") + if not watch_path.exists(): + print(f"error: path not found: {watch_path}", file=sys.stderr) + sys.exit(1) + from graphify.watch import watch as _watch + + try: + _watch(watch_path) + except ImportError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + + elif cmd in ("cluster-only", "label"): + # `label` is `cluster-only` that always (re)generates community names with + # the configured backend, even when a .graphify_labels.json already exists. + force_relabel = cmd == "label" + # Mirror the tree/export arg-parsing pattern: walk argv so flags and + # the optional positional path can appear in any order (#724). + no_viz = "--no-viz" in sys.argv + no_label = "--no-label" in sys.argv + missing_only = "--missing-only" in sys.argv + co_timing = "--timing" in sys.argv + _backend_arg = next((a for a in sys.argv if a.startswith("--backend=")), None) + label_backend = _backend_arg.split("=", 1)[1] if _backend_arg else None + _model_arg = next((a for a in sys.argv if a.startswith("--model=")), None) + label_model = _model_arg.split("=", 1)[1] if _model_arg else None + _min_cs_arg = next((a for a in sys.argv if a.startswith("--min-community-size=")), None) + min_community_size = int(_min_cs_arg.split("=")[1]) if _min_cs_arg else 3 + args = sys.argv[2:] + watch_path: Path | None = None + graph_override: Path | None = None + co_resolution: float = 1.0 + co_exclude_hubs: float | None = None + label_max_concurrency: int = 4 + label_batch_size: int = 100 + i_arg = 0 + while i_arg < len(args): + a = args[i_arg] + if a == "--graph" and i_arg + 1 < len(args): + graph_override = Path(args[i_arg + 1]); i_arg += 2 + elif a == "--backend" and i_arg + 1 < len(args): + label_backend = args[i_arg + 1]; i_arg += 2 + elif a.startswith("--backend="): + label_backend = a.split("=", 1)[1]; i_arg += 1 + elif a == "--model" and i_arg + 1 < len(args): + label_model = args[i_arg + 1]; i_arg += 2 + elif a.startswith("--model="): + label_model = a.split("=", 1)[1]; i_arg += 1 + elif a == "--resolution" and i_arg + 1 < len(args): + co_resolution = float(args[i_arg + 1]); i_arg += 2 + elif a.startswith("--resolution="): + co_resolution = float(a.split("=", 1)[1]); i_arg += 1 + elif a == "--exclude-hubs" and i_arg + 1 < len(args): + co_exclude_hubs = float(args[i_arg + 1]); i_arg += 2 + elif a.startswith("--exclude-hubs="): + co_exclude_hubs = float(a.split("=", 1)[1]); i_arg += 1 + elif a == "--max-concurrency" and i_arg + 1 < len(args): + label_max_concurrency = int(args[i_arg + 1]); i_arg += 2 + elif a.startswith("--max-concurrency="): + label_max_concurrency = int(a.split("=", 1)[1]); i_arg += 1 + elif a == "--batch-size" and i_arg + 1 < len(args): + label_batch_size = int(args[i_arg + 1]); i_arg += 2 + elif a.startswith("--batch-size="): + label_batch_size = int(a.split("=", 1)[1]); i_arg += 1 + elif a in ("--no-viz", "--missing-only") or a.startswith("--min-community-size="): + i_arg += 1 + elif a.startswith("--"): + i_arg += 1 + elif watch_path is None: + watch_path = Path(a); i_arg += 1 + else: + i_arg += 1 + if watch_path is None: + watch_path = Path(".") + graph_json = graph_override if graph_override is not None else watch_path / _GRAPHIFY_OUT / "graph.json" + if not graph_json.exists(): + print( + f"error: no graph found at {graph_json} — run /graphify first", + file=sys.stderr, + ) + sys.exit(1) + from networkx.readwrite import json_graph as _jg + from graphify.build import build_from_json + from graphify.cluster import cluster, score_all, remap_communities_to_previous + from graphify.analyze import ( + god_nodes, + surprising_connections, + suggest_questions, + ) + from graphify.report import generate + from graphify.export import to_json, to_html + + stages = _StageTimer(co_timing) + print("Loading existing graph...") + # Solution 3 (#1019): don't hard-exit on an oversized graph.json here. + # Core outputs (graph.json + GRAPH_REPORT.md) still get written; the + # graph.html render below falls back to the community-aggregation view + # (node_limit=5000) when over the cap. + from graphify.security import check_graph_file_size_cap as _check_cap + _over_cap = False + try: + _check_cap(graph_json) + except ValueError: + _over_cap = True + try: + _over_cap_bytes = graph_json.stat().st_size + except OSError: + _over_cap_bytes = -1 + print( + f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " + f"falling back to community-aggregation view (node_limit=5000)", + file=sys.stderr, + ) + _raw = json.loads(graph_json.read_text(encoding="utf-8")) + _directed = bool(_raw.get("directed", False)) + G = build_from_json(_raw, directed=_directed) + print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") + stages.mark("load") + print("Re-clustering...") + communities = cluster(G, resolution=co_resolution, exclude_hubs_percentile=co_exclude_hubs) + # Mirror the watch/update path (#822): map new cids to prior ones by + # node-overlap so the existing .graphify_labels.json keeps attaching + # to the same conceptual community after re-clustering. Without this, + # labels follow raw cid index and become misaligned whenever the + # graph has changed between labeling and cluster-only (#1027). + previous_node_community = { + n["id"]: n["community"] + for n in _raw.get("nodes", []) + if n.get("community") is not None and n.get("id") is not None + } + if previous_node_community: + communities = remap_communities_to_previous(communities, previous_node_community) + stages.mark("cluster") + cohesion = score_all(G, communities) + gods = god_nodes(G) + surprises = surprising_connections(G, communities) + stages.mark("analyze") + out = watch_path / _GRAPHIFY_OUT + out.mkdir(parents=True, exist_ok=True) + labels_path = out / ".graphify_labels.json" + existing_labels: dict[int, str] = {} + if labels_path.exists(): + try: + existing_labels = { + int(k): v + for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items() + if isinstance(v, str) + } + except Exception: + existing_labels = {} + # Accumulate token usage from the labeling LLM calls so cluster-only mode + # reports real cost instead of a hardcoded zero (#1694). Stays {0, 0} on + # the reuse / no-label paths, which make no LLM calls. + label_token_usage = {"input": 0, "output": 0} + if labels_path.exists() and not force_relabel: + # Reuse saved labels, but don't blindly trust them: the graph may have + # been re-scoped/re-clustered since labeling, in which case a cid now + # covers a DIFFERENT community and its old (LLM) name is wrong (#label-stale). + # Validate each community against the membership signature saved beside the + # labels; any community that changed (or has no saved label) is renamed by + # its current hub — deterministic and correct-by-construction — and the user + # is told to `graphify label` for fresh LLM names. Unchanged communities keep + # their saved label. When no signature sidecar exists (labels predate this), + # fall back to hub-filling only the communities missing a label. + from graphify.cluster import community_member_sigs, label_communities_by_hub + sig_path = labels_path.parent / (labels_path.name + ".sig") + saved_sigs: dict[int, str] = {} + if sig_path.exists(): + try: + saved_sigs = { + int(k): v for k, v in + json.loads(sig_path.read_text(encoding="utf-8")).items() + if isinstance(v, str) + } + except Exception: + saved_sigs = {} + cur_sigs = community_member_sigs(communities) + count_mismatch = len(existing_labels) != len(communities) + labels = {} + hub_labels: dict[int, str] | None = None + changed = 0 + for cid in communities: + have_label = cid in existing_labels + if saved_sigs: + # Precise: the membership signature tells us if this exact + # community changed since it was labeled. + fresh = have_label and saved_sigs.get(cid) == cur_sigs.get(cid) + else: + # No signature sidecar (labels predate it). A differing community + # COUNT means the labels describe a different clustering, so a cid's + # old label can't be trusted; equal count is the best "same" signal. + fresh = have_label and not count_mismatch + if fresh: + labels[cid] = existing_labels[cid] + else: + if hub_labels is None: + hub_labels = label_communities_by_hub(G, communities) + labels[cid] = hub_labels[cid] + if have_label: + changed += 1 + if changed: + print( + f"[graphify] community set changed since labeling " + f"({len(existing_labels)} saved labels, {len(communities)} communities now; " + f"renamed {changed} community(ies) by their hub). " + f"Run `graphify label` to refresh names with the LLM.", + file=sys.stderr, + ) + elif no_label and not force_relabel: + labels = {cid: f"Community {cid}" for cid in communities} + else: + # No labels file yet (or `graphify label` forced a refresh). When run + # standalone there is no orchestrating agent to do skill.md Step 5, so + # auto-name communities rather than leave "Community N" (#1097). + from graphify.cluster import label_communities_by_hub + from graphify.llm import generate_community_labels + print("Labeling communities...") + # Deterministic, LLM-free base labels: name each community after its + # highest-degree hub, so the report is readable even with no backend + # (previously bare "Community N"). A configured LLM backend overrides these + # with richer names below; its no-backend placeholder fallback does NOT. + hub_labels = label_communities_by_hub(G, communities) + label_communities_input = communities + labels = dict(hub_labels) + if missing_only: + labels = { + cid: existing_labels.get(cid, hub_labels[cid]) + for cid in communities + } + label_communities_input = { + cid: members + for cid, members in communities.items() + if cid not in existing_labels or existing_labels.get(cid) == f"Community {cid}" + } + generated_labels, _ = generate_community_labels( + G, label_communities_input, backend=label_backend, model=label_model, gods=gods, + max_concurrency=label_max_concurrency, batch_size=label_batch_size, + usage_out=label_token_usage, + ) + # Only let the LLM OVERRIDE where it produced a real name — its no-backend + # fallback returns "Community {cid}" placeholders, which must not clobber + # the deterministic hub labels. + labels.update({ + cid: v for cid, v in generated_labels.items() + if v and v != f"Community {cid}" + }) + stages.mark("label") + questions = suggest_questions(G, communities, labels) + tokens = label_token_usage + from graphify.export import _git_head as _gh + _commit = _gh() + from graphify.report import load_learning_for_report as _llfr + report = generate(G, communities, cohesion, labels, gods, surprises, + {"warning": "cluster-only mode — file stats not available"}, + tokens, str(watch_path), suggested_questions=questions, + min_community_size=min_community_size, built_at_commit=_commit, + learning=_llfr(out / "graph.json")) + (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") + stages.mark("report") + from graphify.export import backup_if_protected as _backup + _backup(out) + analysis = { + "communities": {str(k): v for k, v in communities.items()}, + "cohesion": {str(k): v for k, v in cohesion.items()}, + "gods": gods, + "surprises": surprises, + "questions": questions, + } + (out / ".graphify_analysis.json").write_text( + json.dumps(analysis, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + to_json(G, communities, str(out / "graph.json"), community_labels=labels) + labels_path.write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding="utf-8") + # Membership signatures beside the labels so a later cluster-only can detect + # which communities changed and avoid reusing a stale label (see reuse above). + from graphify.cluster import community_member_sigs as _cms + (labels_path.parent / (labels_path.name + ".sig")).write_text( + json.dumps({str(k): v for k, v in _cms(communities).items()}), encoding="utf-8") + + # Mirror watch.py pattern: gate to_html so core outputs (graph.json + + # GRAPH_REPORT.md) always land. Honor --no-viz explicitly; otherwise + # fall back to ValueError handling so an oversized graph doesn't crash + # the CLI mid-write and leave a stale graph.html on disk. + html_target = out / "graph.html" + if no_viz: + if html_target.exists(): + html_target.unlink() + stages.mark("export"); stages.total() + print(f"Done - {len(communities)} communities. GRAPH_REPORT.md and graph.json updated (--no-viz; graph.html removed).") + else: + try: + # Over-cap fallback (#1019): force the community-aggregation + # path so an oversized graph still renders a usable graph.html. + _node_limit = 5000 if _over_cap else None + to_html(G, communities, str(html_target), community_labels=labels or None, + node_limit=_node_limit) + stages.mark("export"); stages.total() + print(f"Done - {len(communities)} communities. GRAPH_REPORT.md, graph.json and graph.html updated.") + except ValueError as viz_err: + if html_target.exists(): + html_target.unlink() + print(f"Skipped graph.html: {viz_err}") + stages.mark("export"); stages.total() + print(f"Done - {len(communities)} communities. GRAPH_REPORT.md and graph.json updated.") + + elif cmd == "update": + force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") + no_cluster = False + args = sys.argv[2:] + watch_arg: str | None = None + for a in args: + if a == "--force": + force = True + continue + if a == "--no-cluster": + no_cluster = True + continue + if a.startswith("-"): + print(f"error: unknown update option: {a}", file=sys.stderr) + sys.exit(2) + if watch_arg is not None: + print("error: update accepts at most one path argument", file=sys.stderr) + sys.exit(2) + watch_arg = a + + if watch_arg is not None: + watch_path = Path(watch_arg) + else: + # Try to recover the scan root saved by the last full build + saved = Path(_GRAPHIFY_OUT) / ".graphify_root" + if saved.exists(): + watch_path = Path(saved.read_text(encoding="utf-8").strip()) + else: + watch_path = Path(".") + if not watch_path.exists(): + print(f"error: path not found: {watch_path}", file=sys.stderr) + sys.exit(1) + from graphify.watch import _rebuild_code + + print(f"Re-extracting code files in {watch_path} (no LLM needed)...") + # Interactive CLI: block on the per-repo lock rather than skip, so the + # user sees their explicit `graphify update` complete instead of + # exiting silently when a hook-driven rebuild happens to be running. + ok = _rebuild_code(watch_path, force=force, no_cluster=no_cluster, block_on_lock=True) + if ok: + print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.") + if not ( + os.environ.get("GEMINI_API_KEY") + or os.environ.get("GOOGLE_API_KEY") + or os.environ.get("MOONSHOT_API_KEY") + or os.environ.get("DEEPSEEK_API_KEY") + or os.environ.get("GRAPHIFY_NO_TIPS") + ): + print("Tip: set GEMINI_API_KEY or GOOGLE_API_KEY to use Gemini for semantic extraction.") + else: + print( + "Nothing to update or rebuild failed — check output above.", + file=sys.stderr, + ) + sys.exit(1) + + elif cmd == "hook-check": + # Codex Desktop rejects hookSpecificOutput.additionalContext on PreToolUse. + # Keep this as a cross-platform no-op so installed hooks never break Bash + # tool calls. Graph guidance reaches the agent via AGENTS.md / skill instead. + sys.exit(0) + elif cmd == "hook-guard": + # Shell-agnostic Claude/Codebuddy PreToolUse guard (#522). Replaces the old + # inline-bash hooks that failed on Windows. Prints an additionalContext nudge + # toward graphify when a graph exists; always exits 0 (never blocks a tool). + _run_hook_guard(sys.argv[2] if len(sys.argv) > 2 else "") + sys.exit(0) + elif cmd == "check-update": + if len(sys.argv) < 3: + print("Usage: graphify check-update ", file=sys.stderr) + sys.exit(1) + from graphify.watch import check_update + + check_update(Path(sys.argv[2]).resolve()) + sys.exit(0) + elif cmd == "tree": + # Emit a D3 v7 collapsible-tree HTML view of graph.json: + # expand-all / collapse-all / reset-view buttons, multi-line + # wrapText labels with separately-coloured name + count, + # depth-based palette, click-to-toggle subtree, hover inspector + # showing top-K outbound edges per symbol. + from typing import Optional as _Opt + from graphify.tree_html import write_tree_html, DEFAULT_MAX_CHILDREN + graph_path = Path(_GRAPHIFY_OUT) / "graph.json" + output_path: "_Opt[Path]" = None + root: "_Opt[str]" = None + max_children = DEFAULT_MAX_CHILDREN + top_k_edges = 0 + project_label: "_Opt[str]" = None + args = sys.argv[2:] + i_arg = 0 + while i_arg < len(args): + a = args[i_arg] + if a == "--graph" and i_arg + 1 < len(args): + graph_path = Path(args[i_arg + 1]); i_arg += 2 + elif a == "--output" and i_arg + 1 < len(args): + output_path = Path(args[i_arg + 1]); i_arg += 2 + elif a == "--root" and i_arg + 1 < len(args): + root = args[i_arg + 1]; i_arg += 2 + elif a == "--max-children" and i_arg + 1 < len(args): + max_children = int(args[i_arg + 1]); i_arg += 2 + elif a == "--top-k-edges" and i_arg + 1 < len(args): + top_k_edges = int(args[i_arg + 1]); i_arg += 2 + elif a == "--label" and i_arg + 1 < len(args): + project_label = args[i_arg + 1]; i_arg += 2 + elif a in ("-h", "--help"): + print("Usage: graphify tree [--graph PATH] [--output HTML]") + print(" --graph PATH path to graph.json (default graphify-out/graph.json)") + print(" --output HTML output path (default graphify-out/GRAPH_TREE.html)") + print(" --root PATH filesystem root (default: longest common dir of all source_files)") + print(" --max-children N cap visible children per node (default 200)") + print(" --top-k-edges N pre-compute top-K outbound edges per symbol (default 12)") + print(" --label NAME project label shown in the page header") + return + else: + i_arg += 1 + if not graph_path.is_file(): + print(f"error: graph.json not found at {graph_path}", file=sys.stderr) + sys.exit(1) + _enforce_graph_size_cap_or_exit(graph_path) + if output_path is None: + output_path = graph_path.parent / "GRAPH_TREE.html" + out = write_tree_html( + graph_path=graph_path, output_path=output_path, + root=root, max_children=max_children, + top_k_edges=top_k_edges, project_label=project_label, + ) + size_kb = out.stat().st_size / 1024 + print(f"wrote {out} ({size_kb:.1f} KB)") + print(f"open with: xdg-open {out} (or file://{out.resolve()})") + sys.exit(0) + + elif cmd == "merge-driver": + # git merge driver for graph.json — takes (base, current, other) and writes + # the union of current+other nodes/edges back to current. Exits 1 on + # corrupt input so git surfaces the conflict instead of silently + # accepting a poisoned merge (see F-005). + # Usage: graphify merge-driver %O %A %B (set in .git/config merge driver) + if len(sys.argv) < 5: + print("Usage: graphify merge-driver ", file=sys.stderr) + sys.exit(1) + _base_path, _current_path, _other_path = sys.argv[2], sys.argv[3], sys.argv[4] + # Hard caps so a malicious or corrupted graph.json cannot exhaust memory + # at parse time. 50 MB / 100k nodes are well above any realistic graph + # (typical graphs are <5 MB / <50k nodes); anything larger should fail + # the merge so a human can investigate. + _MERGE_MAX_BYTES = 50 * 1024 * 1024 + _MERGE_MAX_NODES = 100_000 + import networkx as _nx + from networkx.readwrite import json_graph as _jg + def _load_graph(p: str): + path_obj = Path(p) + try: + size = path_obj.stat().st_size + except OSError as exc: + raise RuntimeError(f"cannot stat {p}: {exc}") from exc + if size > _MERGE_MAX_BYTES: + raise RuntimeError( + f"graph.json {p} is {size} bytes, exceeds {_MERGE_MAX_BYTES}-byte cap" + ) + data = json.loads(path_obj.read_text(encoding="utf-8")) + try: + return _jg.node_link_graph(data, edges="links"), data + except TypeError: + return _jg.node_link_graph(data), data + try: + G_cur, _ = _load_graph(_current_path) + G_oth, _ = _load_graph(_other_path) + except Exception as exc: + print(f"[graphify merge-driver] error loading graphs: {exc}", file=sys.stderr) + sys.exit(1) # surface the conflict so git doesn't accept a corrupt merge + merged = _nx.compose(G_cur, G_oth) + if merged.number_of_nodes() > _MERGE_MAX_NODES: + print( + f"[graphify merge-driver] merged graph has {merged.number_of_nodes()} nodes, " + f"exceeds {_MERGE_MAX_NODES}-node cap; aborting merge.", + file=sys.stderr, + ) + sys.exit(1) + try: + out_data = _jg.node_link_data(merged, edges="links") + except TypeError: + out_data = _jg.node_link_data(merged) + Path(_current_path).write_text(json.dumps(out_data, indent=2), encoding="utf-8") + sys.exit(0) + + elif cmd == "merge-graphs": + # graphify merge-graphs graph1.json graph2.json ... --out merged.json + args = sys.argv[2:] + graph_paths: list[Path] = [] + out_path = Path(_GRAPHIFY_OUT) / "merged-graph.json" + i = 0 + while i < len(args): + if args[i] == "--out" and i + 1 < len(args): + out_path = Path(args[i + 1]) + i += 2 + else: + graph_paths.append(Path(args[i])) + i += 1 + if len(graph_paths) < 2: + print( + "Usage: graphify merge-graphs [...] [--out merged.json]", + file=sys.stderr, + ) + sys.exit(1) + import networkx as _nx + from networkx.readwrite import json_graph as _jg + from graphify.build import prefix_graph_for_global as _prefix, distinct_repo_tags as _repo_tags + graphs = [] + for gp in graph_paths: + if not gp.exists(): + print(f"error: not found: {gp}", file=sys.stderr) + sys.exit(1) + _enforce_graph_size_cap_or_exit(gp) + data = json.loads(gp.read_text(encoding="utf-8")) + # Normalize edges/links key before loading — graphify writes "links" + # via node_link_data but older runs may have used "edges" (#738). + if "links" not in data and "edges" in data: + data = dict(data, links=data["edges"]) + try: + G = _jg.node_link_graph(data, edges="links") + except TypeError: + G = _jg.node_link_graph(data) + graphs.append(G) + # nx.compose requires all graphs to be the same type. When input graphs + # come from different sources (e.g. an AST-only run vs a full LLM run) one + # may be a MultiGraph and another a Graph. Normalise everything to Graph + # (the graphify default) by converting MultiGraphs with nx.Graph(). + def _to_simple(g: "_nx.Graph") -> "_nx.Graph": + # nx.compose requires every graph to be the same type. Inputs may + # disagree on BOTH axes — directed vs undirected, and multi vs simple + # — because per-repo graph.json files are written by different extract + # paths at different times. Normalise everything to a plain undirected + # Graph (the merged cross-repo view is undirected anyway), which covers + # DiGraph / MultiGraph / MultiDiGraph. Without this a directed input + # crashed compose with "All graphs must be directed or undirected" (#1606). + if type(g) is not _nx.Graph: + return _nx.Graph(g) + return g + # Unique repo tag per graph. The bare `graphify-out/..` dir name is not + # unique across inputs (src/graphify-out and frontend/src/graphify-out both + # → "src"), which collides same-stem node ids and silently merges unrelated + # entities (#1729). distinct_repo_tags guarantees a distinct prefix per graph. + repo_tags = _repo_tags(graph_paths) + naive_tags = [gp.parent.parent.name for gp in graph_paths] + if len(set(naive_tags)) != len(naive_tags): + print(f" note: repo dir names collide; using distinct tags: {', '.join(repo_tags)}") + merged = _nx.Graph() + for G, repo_tag in zip(graphs, repo_tags): + prefixed = _to_simple(_prefix(G, repo_tag)) + merged = _nx.compose(merged, prefixed) + try: + out_data = _jg.node_link_data(merged, edges="links") + except TypeError: + out_data = _jg.node_link_data(merged) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(out_data, indent=2), encoding="utf-8") + print(f"Merged {len(graphs)} graphs -> {merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges") + print(f"Written to: {out_path}") + + elif cmd == "clone": + if len(sys.argv) < 3: + print( + "Usage: graphify clone [--branch ] [--out ]", + file=sys.stderr, + ) + sys.exit(1) + url = sys.argv[2] + branch: str | None = None + out_dir: Path | None = None + args = sys.argv[3:] + i = 0 + while i < len(args): + if args[i] == "--branch" and i + 1 < len(args): + branch = args[i + 1] + i += 2 + elif args[i] == "--out" and i + 1 < len(args): + out_dir = Path(args[i + 1]) + i += 2 + else: + i += 1 + local_path = _clone_repo(url, branch=branch, out_dir=out_dir) + print(local_path) + + elif cmd == "export": + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd not in ("html", "callflow-html", "obsidian", "wiki", "svg", "graphml", "neo4j", "falkordb"): + print("Usage: graphify export ", file=sys.stderr) + print(" html [--graph PATH] [--labels PATH] [--node-limit N] [--no-viz]", file=sys.stderr) + print(" callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH] [--report PATH] [--sections PATH] [--output HTML]", file=sys.stderr) + print(" [--lang auto|zh-CN|en] [--max-sections N] [--diagram-scale N]", file=sys.stderr) + print(" obsidian [--graph PATH] [--labels PATH] [--dir PATH]", file=sys.stderr) + print(" wiki [--graph PATH] [--labels PATH]", file=sys.stderr) + print(" svg [--graph PATH] [--labels PATH]", file=sys.stderr) + print(" graphml [--graph PATH]", file=sys.stderr) + print(" neo4j [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) + print(" (or set NEO4J_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) + print(" falkordb [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) + print(" (or set FALKORDB_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) + sys.exit(1) + + # Parse shared args + args = sys.argv[3:] + graph_path = Path(_GRAPHIFY_OUT) / "graph.json" + graph_path_explicit = False + labels_path = Path(_GRAPHIFY_OUT) / ".graphify_labels.json" + labels_path_explicit = False + report_path = Path(_GRAPHIFY_OUT) / "GRAPH_REPORT.md" + report_path_explicit = False + sections_path: Path | None = None + callflow_output: Path | None = None + callflow_lang = "auto" + callflow_max_sections = 15 + callflow_diagram_scale = 1.0 + callflow_max_diagram_nodes = 18 + callflow_max_diagram_edges = 24 + analysis_path = Path(_GRAPHIFY_OUT) / ".graphify_analysis.json" + node_limit = 5000 + no_viz = False + obsidian_dir = Path(_GRAPHIFY_OUT) / "obsidian" + # Shared push-connection settings for the graph-database sinks (neo4j, + # falkordb), parsed from the generic --push/--user/--password flags below. + push_uri: str | None = None + push_user = "neo4j" # Neo4j default user; FalkorDB auth is optional and ignores it + # F-031: prefer an env var so the password never appears on argv (visible + # in `ps` output / shell history). The explicit --password flag still + # overrides it. Each sink reads its own var: FALKORDB_PASSWORD for falkordb, + # NEO4J_PASSWORD otherwise. + push_password: str | None = ( + os.environ.get("FALKORDB_PASSWORD") if subcmd == "falkordb" + else os.environ.get("NEO4J_PASSWORD") + ) or None + i = 0 + while i < len(args): + a = args[i] + if a == "--graph" and i + 1 < len(args): + graph_path = Path(args[i + 1]) + graph_path_explicit = True + i += 2 + elif a == "--labels" and i + 1 < len(args): + labels_path = Path(args[i + 1]) + labels_path_explicit = True + i += 2 + elif a == "--report" and i + 1 < len(args): + report_path = Path(args[i + 1]) + report_path_explicit = True + i += 2 + elif a == "--sections" and i + 1 < len(args): + sections_path = Path(args[i + 1]); i += 2 + elif a == "--output" and i + 1 < len(args): + callflow_output = Path(args[i + 1]).expanduser() + if not callflow_output.is_absolute(): + callflow_output = Path.cwd() / callflow_output + i += 2 + elif a == "--lang" and i + 1 < len(args): + callflow_lang = args[i + 1]; i += 2 + elif a == "--max-sections" and i + 1 < len(args): + callflow_max_sections = int(args[i + 1]); i += 2 + elif a == "--diagram-scale" and i + 1 < len(args): + callflow_diagram_scale = float(args[i + 1]); i += 2 + elif a == "--max-diagram-nodes" and i + 1 < len(args): + callflow_max_diagram_nodes = int(args[i + 1]); i += 2 + elif a == "--max-diagram-edges" and i + 1 < len(args): + callflow_max_diagram_edges = int(args[i + 1]); i += 2 + elif a in ("-h", "--help") and subcmd == "callflow-html": + print("Usage: graphify export callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH]") + print(" --report PATH path to GRAPH_REPORT.md") + print(" --sections PATH JSON section definitions") + print(" --output HTML output path (default graphify-out/-callflow.html)") + print(" --lang LANG auto, zh-CN, en, etc. (default auto)") + print(" --max-sections N maximum auto-derived sections (default 15)") + print(" --diagram-scale N Mermaid diagram scale (default 1.0)") + print(" --max-diagram-nodes N representative nodes per section (default 18)") + print(" --max-diagram-edges N representative edges per section (default 24)") + sys.exit(0) + elif a == "--node-limit" and i + 1 < len(args): + node_limit = int(args[i + 1]); i += 2 + elif a == "--no-viz": + no_viz = True; i += 1 + elif a == "--dir" and i + 1 < len(args): + obsidian_dir = Path(args[i + 1]); i += 2 + elif a == "--push" and i + 1 < len(args): + push_uri = args[i + 1]; i += 2 + elif a == "--user" and i + 1 < len(args): + push_user = args[i + 1]; i += 2 + elif a == "--password" and i + 1 < len(args): + push_password = args[i + 1]; i += 2 + elif subcmd == "callflow-html" and not a.startswith("-") and not graph_path_explicit: + candidate = Path(a) + if candidate.name == "graph.json" or candidate.suffix.lower() == ".json": + graph_path = candidate + elif (candidate / "graph.json").exists(): + graph_path = candidate / "graph.json" + else: + graph_path = candidate / _GRAPHIFY_OUT / "graph.json" + graph_path_explicit = True + i += 1 + else: + i += 1 + + graph_path = graph_path.expanduser() + if graph_path_explicit: + graph_out_dir = graph_path.parent + if not labels_path_explicit: + labels_path = graph_out_dir / ".graphify_labels.json" + if not report_path_explicit: + report_path = graph_out_dir / "GRAPH_REPORT.md" + labels_path = labels_path.expanduser() + report_path = report_path.expanduser() + + if not graph_path.exists(): + print(f"error: graph not found: {graph_path}. Run /graphify first.", file=sys.stderr) + sys.exit(1) + + if subcmd == "callflow-html": + from graphify.callflow_html import write_callflow_html as _write_callflow_html + out = _write_callflow_html( + graph=graph_path, + report=report_path, + labels=labels_path, + sections=sections_path, + output=callflow_output, + lang=callflow_lang, + max_sections=callflow_max_sections, + diagram_scale=callflow_diagram_scale, + max_diagram_nodes=callflow_max_diagram_nodes, + max_diagram_edges=callflow_max_diagram_edges, + verbose=True, + ) + print(f"callflow HTML written - open in any browser: {out}") + sys.exit(0) + + from networkx.readwrite import json_graph as _jg + from graphify.build import build_from_json as _bfj + from graphify.security import check_graph_file_size_cap as _check_cap + + # Solution 3 (#1019): for the HTML view, an oversized graph.json should + # not be a hard error. Detect the over-cap condition here and fall back + # to the community-aggregation view (node_limit=5000) below instead of + # exiting 1. All other subcommands keep the hard cap. + _over_cap = False + try: + _check_cap(graph_path) + except ValueError as _cap_err: + if subcmd == "html": + _over_cap = True + try: + _over_cap_bytes = graph_path.stat().st_size + except OSError: + _over_cap_bytes = -1 + print( + f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " + f"falling back to community-aggregation view (node_limit=5000)", + file=sys.stderr, + ) + else: + print(f"error: {_cap_err}", file=sys.stderr) + sys.exit(1) + _raw = json.loads(graph_path.read_text(encoding="utf-8")) + if "links" not in _raw and "edges" in _raw: + _raw = dict(_raw, links=_raw["edges"]) + try: + G = _jg.node_link_graph(_raw, edges="links") + except TypeError: + G = _jg.node_link_graph(_raw) + + # Load optional analysis/labels + communities: dict[int, list[str]] = {} + if analysis_path.exists(): + _an = json.loads(analysis_path.read_text(encoding="utf-8")) + communities = {int(k): v for k, v in _an.get("communities", {}).items()} + cohesion: dict[int, float] = {int(k): v for k, v in _an.get("cohesion", {}).items()} + gods_data = _an.get("gods", []) + else: + cohesion = {} + gods_data = [] + + # Fallback: graph.json carries the per-node community as a node attribute + # (`to_json` writes it on every node). The analysis sidecar is the + # canonical source — but the post-commit / watch rebuild path doesn't + # regenerate it, and `extract` may have its temp files cleaned up. When + # that happens, `graphify export html` previously bailed with + # "Single community - aggregated view not useful." even though the + # per-node attribute had the right data all along. Reconstruct from + # the graph itself so downstream subcommands (html, obsidian, wiki, + # svg, graphml, neo4j) don't silently produce a degraded artifact. + if not communities: + reconstructed: dict[int, list[str]] = {} + for node_id, data in G.nodes(data=True): + cid_raw = data.get("community") + if cid_raw is None: + continue + try: + cid = int(cid_raw) + except (TypeError, ValueError): + continue + reconstructed.setdefault(cid, []).append(str(node_id)) + if reconstructed: + communities = reconstructed + + labels: dict[int, str] = {} + if labels_path.exists(): + labels = {int(k): v for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items()} + + out_dir = graph_path.parent + + if subcmd == "html": + from graphify.export import to_html as _to_html + if no_viz: + html_target = out_dir / "graph.html" + if html_target.exists(): + html_target.unlink() + print("--no-viz: skipped graph.html") + else: + # Over-cap fallback (#1019): force the community-aggregation + # path so the oversized graph still renders a usable artifact. + _effective_node_limit = 5000 if _over_cap else node_limit + _to_html(G, communities, str(out_dir / "graph.html"), + community_labels=labels or None, node_limit=_effective_node_limit) + if G.number_of_nodes() <= _effective_node_limit: + print(f"graph.html written - open in any browser, no server needed") + if _over_cap: + sys.exit(0) + + elif subcmd == "obsidian": + from graphify.export import to_obsidian as _to_obsidian, to_canvas as _to_canvas + n = _to_obsidian(G, communities, str(obsidian_dir), + community_labels=labels or None, cohesion=cohesion or None) + print(f"Obsidian vault: {n} notes in {obsidian_dir}/") + _to_canvas(G, communities, str(obsidian_dir / "graph.canvas"), + community_labels=labels or None) + print(f"Canvas: {obsidian_dir}/graph.canvas") + print(f"Open {obsidian_dir}/ as a vault in Obsidian.") + + elif subcmd == "wiki": + from graphify.wiki import to_wiki as _to_wiki + from graphify.analyze import god_nodes as _god_nodes + if not communities: + print( + "error: .graphify_analysis.json is missing or empty — refusing to export wiki to prevent data loss.\n" + "Run `graphify extract .` (or `graphify cluster-only .`) to regenerate community data first.", + file=sys.stderr, + ) + sys.exit(1) + if not gods_data: + gods_data = _god_nodes(G) + n = _to_wiki(G, communities, str(out_dir / "wiki"), + community_labels=labels or None, cohesion=cohesion or None, + god_nodes_data=gods_data) + print(f"Wiki: {n} articles written to {out_dir}/wiki/") + print(f" {out_dir}/wiki/index.md -> agent entry point") + + elif subcmd == "svg": + from graphify.export import to_svg as _to_svg + _to_svg(G, communities, str(out_dir / "graph.svg"), + community_labels=labels or None) + print(f"graph.svg written - embeds in Obsidian, Notion, GitHub READMEs") + + elif subcmd == "graphml": + from graphify.export import to_graphml as _to_graphml + _to_graphml(G, communities, str(out_dir / "graph.graphml")) + print(f"graph.graphml written - open in Gephi, yEd, or any GraphML tool") + + elif subcmd == "neo4j": + if push_uri: + from graphify.export import push_to_neo4j as _push + if push_password is None: + print("error: --password required for --push", file=sys.stderr) + sys.exit(1) + result = _push(G, uri=push_uri, user=push_user, + password=push_password, communities=communities) + print(f"Pushed to Neo4j: {result['nodes']} nodes, {result['edges']} edges") + else: + from graphify.export import to_cypher as _to_cypher + _to_cypher(G, str(out_dir / "cypher.txt")) + print(f"cypher.txt written - import with: cypher-shell < {out_dir}/cypher.txt") + + elif subcmd == "falkordb": + if push_uri: + from graphify.export import push_to_falkordb as _push + result = _push(G, uri=push_uri, user=push_user, + password=push_password, communities=communities) + print(f"Pushed to FalkorDB: {result['nodes']} nodes, {result['edges']} edges") + else: + from graphify.export import to_cypher as _to_cypher + _to_cypher(G, str(out_dir / "cypher.txt")) + print(f"cypher.txt written ({out_dir}/cypher.txt) - statements are OpenCypher. " + f"FalkorDB's GRAPH.QUERY runs one statement at a time (no bulk script " + f"import), so load a graph with: graphify export falkordb --push " + f"falkordb://localhost:6379") + + elif cmd == "benchmark": + from graphify.benchmark import run_benchmark, print_benchmark + + graph_path = sys.argv[2] if len(sys.argv) > 2 else _default_graph_path() + _enforce_graph_size_cap_or_exit(Path(graph_path)) + # Try to load corpus_words from detect output + corpus_words = None + detect_path = Path(".graphify_detect.json") + if detect_path.exists(): + try: + detect_data = json.loads(detect_path.read_text(encoding="utf-8")) + corpus_words = detect_data.get("total_words") + except Exception: + pass + result = run_benchmark(graph_path, corpus_words=corpus_words) + print_benchmark(result) + + elif cmd == "global": + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + from graphify.global_graph import ( + global_add as _global_add, + global_remove as _global_remove, + global_list as _global_list, + global_path as _global_path, + ) + if subcmd == "add": + # graphify global add [--as ] + args = sys.argv[3:] + source = None + tag = None + i = 0 + while i < len(args): + if args[i] == "--as" and i + 1 < len(args): + tag = args[i + 1]; i += 2 + elif not source: + source = Path(args[i]); i += 1 + else: + i += 1 + if not source: + print("Usage: graphify global add [--as ]", file=sys.stderr) + sys.exit(1) + tag = tag or source.parent.parent.name + try: + result = _global_add(source, tag) + if result["skipped"]: + print(f"'{tag}' unchanged since last add - global graph not modified.") + else: + print(f"Added '{tag}' to global graph: +{result['nodes_added']} nodes, " + f"-{result['nodes_removed']} pruned. Global: {_global_path()}") + except Exception as exc: + print(f"error: {exc}", file=sys.stderr); sys.exit(1) + elif subcmd == "remove": + tag = sys.argv[3] if len(sys.argv) > 3 else "" + if not tag: + print("Usage: graphify global remove ", file=sys.stderr); sys.exit(1) + try: + removed = _global_remove(tag) + print(f"Removed '{tag}' from global graph ({removed} nodes pruned).") + except KeyError as exc: + print(f"error: {exc}", file=sys.stderr); sys.exit(1) + elif subcmd == "list": + repos = _global_list() + if not repos: + print("Global graph is empty. Use 'graphify global add' to add a project.") + else: + print(f"Global graph: {_global_path()}") + for tag, info in repos.items(): + print(f" {tag}: {info.get('node_count', '?')} nodes, added {info.get('added_at', '?')[:10]}") + elif subcmd == "path": + print(_global_path()) + else: + print("Usage: graphify global [add|remove|list|path]", file=sys.stderr); sys.exit(1) + + elif cmd == "extract": + # Headless full-pipeline extraction for CI / scripts (#698). + # Runs detect -> AST extraction on code -> semantic LLM extraction on + # docs/papers/images -> merge -> build -> cluster -> write outputs. + # Unlike the skill.md path (which runs through Claude Code subagents), + # this calls extract_corpus_parallel directly using whichever backend + # has an API key set. + if len(sys.argv) < 3: + print( + "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " + "[--model M] [--mode deep] [--out DIR] [--google-workspace] [--no-cluster] " + "[--max-workers N] [--token-budget N] [--max-concurrency N] " + "[--api-timeout S] [--postgres DSN] [--cargo] [--timing]", + file=sys.stderr, + ) + sys.exit(1) + + has_path = True + if sys.argv[2].startswith("-"): + has_path = False + target = Path(".").resolve() + else: + target = Path(sys.argv[2]).resolve() + if not target.exists(): + print(f"error: path not found: {target}", file=sys.stderr) + sys.exit(1) + + backend: str | None = None + model: str | None = None + extract_mode: str | None = None + out_dir: Path | None = None + cli_postgres_dsn: str | None = None + cli_cargo: bool = False + no_cluster = False + dedup_llm = False + google_workspace = False + global_merge = False + code_only = False + global_repo_tag: str | None = None + # Performance/tuning knobs (issue #792). None means "use library default". + cli_max_workers: int | None = None + cli_token_budget: int | None = None + cli_max_concurrency: int | None = None + cli_api_timeout: float | None = None + # Clustering tuning knobs + cli_resolution: float = 1.0 + cli_exclude_hubs: float | None = None + cli_excludes: list[str] = [] + cli_timing: bool = False + + def _parse_int(name: str, raw: str) -> int: + try: + v = int(raw) + except ValueError: + print(f"error: {name} must be a positive integer (got {raw!r})", file=sys.stderr) + sys.exit(2) + if v <= 0: + print(f"error: {name} must be > 0 (got {v})", file=sys.stderr) + sys.exit(2) + return v + + def _parse_float(name: str, raw: str) -> float: + try: + v = float(raw) + except ValueError: + print(f"error: {name} must be a positive number (got {raw!r})", file=sys.stderr) + sys.exit(2) + if v <= 0: + print(f"error: {name} must be > 0 (got {v})", file=sys.stderr) + sys.exit(2) + return v + + args = sys.argv[3:] if has_path else sys.argv[2:] + i = 0 + while i < len(args): + a = args[i] + if a == "--backend" and i + 1 < len(args): + backend = args[i + 1]; i += 2 + elif a.startswith("--backend="): + backend = a.split("=", 1)[1]; i += 1 + elif a == "--model" and i + 1 < len(args): + model = args[i + 1]; i += 2 + elif a.startswith("--model="): + model = a.split("=", 1)[1]; i += 1 + elif a == "--mode" and i + 1 < len(args): + extract_mode = args[i + 1]; i += 2 + elif a.startswith("--mode="): + extract_mode = a.split("=", 1)[1]; i += 1 + elif a == "--out" and i + 1 < len(args): + out_dir = Path(args[i + 1]); i += 2 + elif a.startswith("--out="): + out_dir = Path(a.split("=", 1)[1]); i += 1 + elif a == "--no-cluster": + no_cluster = True; i += 1 + elif a == "--dedup-llm": + dedup_llm = True; i += 1 + elif a == "--code-only": + code_only = True; i += 1 + elif a == "--google-workspace": + google_workspace = True; i += 1 + elif a == "--global": + global_merge = True; i += 1 + elif a == "--as" and i + 1 < len(args): + global_repo_tag = args[i + 1]; i += 2 + elif a == "--max-workers" and i + 1 < len(args): + cli_max_workers = _parse_int("--max-workers", args[i + 1]); i += 2 + elif a.startswith("--max-workers="): + cli_max_workers = _parse_int("--max-workers", a.split("=", 1)[1]); i += 1 + elif a == "--token-budget" and i + 1 < len(args): + cli_token_budget = _parse_int("--token-budget", args[i + 1]); i += 2 + elif a.startswith("--token-budget="): + cli_token_budget = _parse_int("--token-budget", a.split("=", 1)[1]); i += 1 + elif a == "--max-concurrency" and i + 1 < len(args): + cli_max_concurrency = _parse_int("--max-concurrency", args[i + 1]); i += 2 + elif a.startswith("--max-concurrency="): + cli_max_concurrency = _parse_int("--max-concurrency", a.split("=", 1)[1]); i += 1 + elif a == "--api-timeout" and i + 1 < len(args): + cli_api_timeout = _parse_float("--api-timeout", args[i + 1]); i += 2 + elif a.startswith("--api-timeout="): + cli_api_timeout = _parse_float("--api-timeout", a.split("=", 1)[1]); i += 1 + elif a == "--resolution" and i + 1 < len(args): + cli_resolution = _parse_float("--resolution", args[i + 1]); i += 2 + elif a.startswith("--resolution="): + cli_resolution = _parse_float("--resolution", a.split("=", 1)[1]); i += 1 + elif a == "--exclude-hubs" and i + 1 < len(args): + cli_exclude_hubs = float(args[i + 1]); i += 2 + elif a.startswith("--exclude-hubs="): + cli_exclude_hubs = float(a.split("=", 1)[1]); i += 1 + elif a == "--exclude" and i + 1 < len(args): + cli_excludes.append(args[i + 1]); i += 2 + elif a.startswith("--exclude="): + cli_excludes.append(a.split("=", 1)[1]); i += 1 + elif a == "--postgres" and i + 1 < len(args): + cli_postgres_dsn = args[i + 1]; i += 2 + elif a.startswith("--postgres="): + cli_postgres_dsn = a.split("=", 1)[1]; i += 1 + elif a == "--cargo": + cli_cargo = True + i += 1 + elif a == "--timing": + cli_timing = True; i += 1 + else: + i += 1 + + if not has_path and cli_postgres_dsn is None: + print("error: must specify a path to scan or a --postgres DSN", file=sys.stderr) + sys.exit(1) + + _VALID_MODES = {"deep"} + if extract_mode is not None and extract_mode not in _VALID_MODES: + print( + f"error: unknown --mode '{extract_mode}'. " + f"Available: {', '.join(sorted(_VALID_MODES))}", + file=sys.stderr, + ) + sys.exit(2) + deep_mode = extract_mode == "deep" + if deep_mode: + print("[graphify extract] deep mode enabled: richer semantic extraction") + + # CLI flag wins over env var. Setting GRAPHIFY_API_TIMEOUT here so + # _call_openai_compat picks it up without needing a new kwarg path. + if cli_api_timeout is not None: + os.environ["GRAPHIFY_API_TIMEOUT"] = str(cli_api_timeout) + if cli_max_workers is not None: + os.environ["GRAPHIFY_MAX_WORKERS"] = str(cli_max_workers) + + # Resolve output dir. The user-facing contract is "/graphify-out/" + # so a fresh checkout writes graphify-out/ at the project root, matching + # the skill.md pipeline. + out_root = (out_dir.resolve() if out_dir else target) + graphify_out = out_root / _GRAPHIFY_OUT + graphify_out.mkdir(parents=True, exist_ok=True) + + stages = _StageTimer(cli_timing) + + from graphify.detect import ( + detect as _detect, + detect_incremental as _detect_incremental, + save_manifest as _save_manifest, + ) + manifest_path = graphify_out / "manifest.json" + existing_graph_path = graphify_out / "graph.json" + incremental_mode = manifest_path.exists() and existing_graph_path.exists() if has_path else False + + if not has_path: + code_files = [] + doc_files = [] + paper_files = [] + image_files = [] + deleted_files = [] + unchanged_total = 0 + files_by_type = {} + elif incremental_mode: + print(f"[graphify extract] incremental scan of {target}") + detection = _detect_incremental( + target, + manifest_path=str(manifest_path), + google_workspace=google_workspace or None, + extra_excludes=cli_excludes or None, + ) + files_by_type = detection.get("files", {}) + new_by_type = detection.get("new_files", {}) + code_files = [Path(p) for p in new_by_type.get("code", [])] + doc_files = [Path(p) for p in new_by_type.get("document", [])] + paper_files = [Path(p) for p in new_by_type.get("paper", [])] + image_files = [Path(p) for p in new_by_type.get("image", [])] + deleted_files = list(detection.get("deleted_files", [])) + unchanged_total = sum(len(v) for v in detection.get("unchanged_files", {}).values()) + else: + print(f"[graphify extract] scanning {target}") + detection = _detect(target, google_workspace=google_workspace or None, extra_excludes=cli_excludes or None) + files_by_type = detection.get("files", {}) + code_files = [Path(p) for p in files_by_type.get("code", [])] + doc_files = [Path(p) for p in files_by_type.get("document", [])] + paper_files = [Path(p) for p in files_by_type.get("paper", [])] + image_files = [Path(p) for p in files_by_type.get("image", [])] + deleted_files = [] + unchanged_total = 0 + + semantic_files = doc_files + paper_files + image_files + # --code-only: index code (pure local AST, no key) and skip the semantic + # (doc/paper/image) pass entirely, so a mixed repo doesn't hard-fail when no + # LLM backend is configured (#1734). Report what was skipped rather than + # silently dropping it. + if code_only and semantic_files: + print( + f"[graphify extract] --code-only: skipping {len(semantic_files)} " + f"non-code file(s) ({len(doc_files)} docs, {len(paper_files)} papers, " + f"{len(image_files)} images) — no LLM extraction" + ) + semantic_files = [] + doc_files = [] + paper_files = [] + image_files = [] + if incremental_mode: + print( + f"[graphify extract] {len(code_files)} code, {len(doc_files)} docs, " + f"{len(paper_files)} papers, {len(image_files)} images changed; " + f"{unchanged_total} unchanged; {len(deleted_files)} deleted" + ) + else: + print( + f"[graphify extract] found {len(code_files)} code, " + f"{len(doc_files)} docs, {len(paper_files)} papers, " + f"{len(image_files)} images" + ) + # Surface files that were seen but not classified (extensionless non-shebang + # project files like Dockerfile/Makefile, or unsupported extensions), so they + # are no longer invisible in graphify's own output (#1692). + _unclassified = detection.get("unclassified", []) if isinstance(detection, dict) else [] + if _unclassified: + _names = ", ".join(sorted({Path(p).name for p in _unclassified})[:6]) + _more = f" (+{len(_unclassified) - 6} more)" if len(_unclassified) > 6 else "" + print( + f"[graphify extract] {len(_unclassified)} file(s) not classified " + f"(no supported extension or shebang), skipped: {_names}{_more}" + ) + stages.mark("detect") + + # Resolve the LLM backend only now that we know whether the corpus + # needs one. A code-only corpus is pure local AST and must not require + # an API key; the key is enforced below only when there's LLM work. + from graphify.llm import ( + BACKENDS as _BACKENDS, + detect_backend as _detect_backend, + estimate_cost as _estimate_cost, + extract_corpus_parallel as _extract_corpus_parallel, + _format_backend_env_keys, + _get_backend_api_key, + ) + needs_llm = bool(semantic_files) or dedup_llm + if backend is None and needs_llm: + backend = _detect_backend() + if backend is not None and backend not in _BACKENDS: + print( + f"error: unknown backend '{backend}'. " + f"Available: {', '.join(sorted(_BACKENDS))}", + file=sys.stderr, + ) + sys.exit(1) + if needs_llm: + if backend is None: + reasons = [] + if semantic_files: + reasons.append( + f"{len(semantic_files)} doc/paper/image file(s) need semantic extraction" + ) + if dedup_llm: + reasons.append("--dedup-llm was passed") + hint = "" + if semantic_files: + hint = (" Or pass --code-only to index just the code " + "(local AST, no key) and skip the non-code files.") + print( + "error: no LLM API key found (" + "; ".join(reasons) + "). " + "Set GEMINI_API_KEY or GOOGLE_API_KEY (gemini), MOONSHOT_API_KEY " + "(kimi), ANTHROPIC_API_KEY (claude), OPENAI_API_KEY (openai), " + "DEEPSEEK_API_KEY (deepseek), or pass --backend. A code-only " + "corpus needs no key." + hint, + file=sys.stderr, + ) + sys.exit(1) + if backend == "ollama": + from graphify.llm import _validate_ollama_base_url + _oll_url = os.environ.get("OLLAMA_BASE_URL", _BACKENDS["ollama"].get("base_url", "")) + try: + _validate_ollama_base_url(_oll_url, warn=False) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(2) + if not _get_backend_api_key(backend): + allow_no_key = False + if backend == "ollama": + from urllib.parse import urlparse + ollama_url = os.environ.get( + "OLLAMA_BASE_URL", + _BACKENDS["ollama"].get("base_url", ""), + ) + try: + host = (urlparse(ollama_url).hostname or "").lower() + except Exception: + host = "" + allow_no_key = ( + host in ("localhost", "127.0.0.1", "::1") + or host.startswith("127.") + ) + elif backend == "bedrock": + allow_no_key = bool( + os.environ.get("AWS_PROFILE") + or os.environ.get("AWS_REGION") + or os.environ.get("AWS_DEFAULT_REGION") + or os.environ.get("AWS_ACCESS_KEY_ID") + ) + elif backend == "claude-cli": + import shutil as _shutil + allow_no_key = _shutil.which("claude") is not None + if not allow_no_key: + print( + "error: backend 'claude-cli' requires the `claude` CLI on $PATH " + "(install Claude Code and run `claude` once to authenticate).", + file=sys.stderr, + ) + sys.exit(1) + if not allow_no_key: + print( + f"error: backend '{backend}' requires {_format_backend_env_keys(backend)} to be set.", + file=sys.stderr, + ) + sys.exit(1) + + # AST extraction on code files. Empty code list (docs-only corpus) is + # the issue #698 case — skip cleanly instead of crashing inside extract(). + ast_result: dict = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} + if code_files: + from graphify.extract import extract as _ast_extract + # Anchor the cache at the output root, not the scanned project: + # with --out, a /graphify-out/cache/ would leak a + # graphify-out/ dir into a project that asked for external output. + ast_kwargs: dict = {"cache_root": out_root} + if cli_max_workers is not None: + ast_kwargs["max_workers"] = cli_max_workers + print(f"[graphify extract] AST extraction on {len(code_files)} code files...") + try: + ast_result = _ast_extract(code_files, **ast_kwargs) + except Exception as exc: + print(f"[graphify extract] AST extraction failed: {exc}", file=sys.stderr) + ast_result = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} + stages.mark("AST extract") + + # Semantic extraction on docs/papers/images. Check cache first. + from graphify.cache import ( + check_semantic_cache as _check_semantic_cache, + prune_semantic_cache as _prune_semantic_cache, + save_semantic_cache as _save_semantic_cache, + ) + sem_result: dict = { + "nodes": [], "edges": [], "hyperedges": [], + "input_tokens": 0, "output_tokens": 0, + } + sem_cache_hits = 0 + sem_cache_misses = 0 + if semantic_files: + sem_paths_str = [str(p) for p in semantic_files] + cached_nodes, cached_edges, cached_hyperedges, uncached_paths = ( + _check_semantic_cache(sem_paths_str, root=out_root) + ) + sem_cache_hits = len(semantic_files) - len(uncached_paths) + sem_cache_misses = len(uncached_paths) + sem_result["nodes"].extend(cached_nodes) + sem_result["edges"].extend(cached_edges) + sem_result["hyperedges"].extend(cached_hyperedges) + if sem_cache_hits: + print(f"[graphify extract] semantic cache: {sem_cache_hits} hit / {sem_cache_misses} miss") + + if uncached_paths: + print(f"[graphify extract] semantic extraction on {len(uncached_paths)} files via {backend}...") + corpus_kwargs: dict = { + "backend": backend, + "model": model, + "root": target, + } + if deep_mode: + corpus_kwargs["deep_mode"] = True + if cli_token_budget is not None: + corpus_kwargs["token_budget"] = cli_token_budget + if cli_max_concurrency is not None: + corpus_kwargs["max_concurrency"] = cli_max_concurrency + + # Minimal progress callback so the CLI is no longer silent + # during long local-inference runs (issue #792 addendum). + # Also track per-chunk success so we can fail loudly when + # every chunk errors (e.g. missing backend SDK package). + _chunk_stats = {"total": 0, "succeeded": 0} + def _progress(idx: int, total: int, _result: dict) -> None: + _chunk_stats["total"] = total + _chunk_stats["succeeded"] += 1 + print( + f"[graphify extract] chunk {idx + 1}/{total} done", + flush=True, + ) + corpus_kwargs["on_chunk_done"] = _progress + + try: + fresh = _extract_corpus_parallel( + [Path(p) for p in uncached_paths], + **corpus_kwargs, + ) + except ImportError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + except Exception as exc: + print( + f"[graphify extract] semantic extraction failed: {exc}", + file=sys.stderr, + ) + fresh = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} + + # on_chunk_done only fires after a chunk succeeds. If fresh + # semantic extraction was requested and no chunks completed, + # fail instead of writing an AST-only graph with exit 0. + if uncached_paths and _chunk_stats["succeeded"] == 0: + print( + f"[graphify extract] error: all semantic chunks failed " + f"for backend '{backend}' ({len(uncached_paths)} uncached files) - " + f"see per-chunk errors above. If you see 'requires the X package', " + f"run `pip install X` and retry.", + file=sys.stderr, + ) + sys.exit(1) + try: + _save_semantic_cache( + fresh.get("nodes", []), + fresh.get("edges", []), + fresh.get("hyperedges", []), + root=out_root, + ) + except Exception as exc: + print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr) + sem_result["nodes"].extend(fresh.get("nodes", [])) + sem_result["edges"].extend(fresh.get("edges", [])) + sem_result["hyperedges"].extend(fresh.get("hyperedges", [])) + sem_result["input_tokens"] += fresh.get("input_tokens", 0) + sem_result["output_tokens"] += fresh.get("output_tokens", 0) + + # Prune orphaned semantic cache entries. The semantic cache is + # content-hash-keyed and unversioned, so it is never swept by the AST + # version-cleanup: every content change or file deletion leaves a + # permanent orphan that accumulates unbounded (#1527). Sweep it against + # the FULL live document set (``files_by_type`` — present in both the + # incremental and full branches), NOT the incremental ``semantic_files`` + # changed-subset, which would delete every unchanged doc's valid entry. + # Best-effort: a prune failure must never break extraction. + try: + from graphify.cache import file_hash as _file_hash + _live_hashes: set[str] = set() + for _kind in ("document", "paper", "image"): + for _fp in files_by_type.get(_kind, []): + _abs = Path(_fp) + if not _abs.is_absolute(): + _abs = Path(out_root) / _abs + if not _abs.is_file(): + continue # deleted/missing — leave out so its entry is pruned + try: + _live_hashes.add(_file_hash(_abs, out_root)) + except OSError: + pass + _prune_semantic_cache(out_root, _live_hashes) + except Exception as exc: + print(f"[graphify extract] warning: could not prune semantic cache: {exc}", file=sys.stderr) + stages.mark("semantic extract") + + pg_result: dict = {"nodes": [], "edges": []} + if cli_postgres_dsn is not None: + from graphify.pg_introspect import introspect_postgres + print(f"[graphify extract] introspecting PostgreSQL schema...") + try: + pg_result = introspect_postgres(cli_postgres_dsn) + except (ConnectionError, ImportError) as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + print(f"[graphify extract] PostgreSQL: {len(pg_result['nodes'])} nodes, " + f"{len(pg_result['edges'])} edges") + + cargo_result: dict = {"nodes": [], "edges": []} + if cli_cargo: + from graphify.cargo_introspect import introspect_cargo + print("[graphify extract] introspecting Cargo workspace...") + try: + cargo_result = introspect_cargo(target) + except (ConnectionError, ImportError, OSError) as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + print(f"[graphify extract] Cargo: {len(cargo_result['nodes'])} nodes, " + f"{len(cargo_result['edges'])} edges") + + # Merge AST + semantic + pg_result + cargo_result. Order matters for deduplication: passing AST + # first means semantic node attributes win on collision (richer labels + # for symbols also referenced in docs). Hyperedges only come from the + # semantic side. + merged: dict = { + "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])) + list(cargo_result.get("nodes", [])), + "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])) + list(cargo_result.get("edges", [])), + "hyperedges": list(sem_result.get("hyperedges", [])), + "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0), + "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0), + } + + graph_json_path = graphify_out / "graph.json" + analysis_path = graphify_out / ".graphify_analysis.json" + + # Build a manifest-safe files dict: only stamp semantic_hash for files + # that actually produced output (cache hit or fresh extraction). Files + # whose chunk failed have no source_file entry in sem_result — leaving + # their semantic_hash empty so detect_incremental re-queues them (#933). + _sem_extracted: set[str] = { + n.get("source_file", "") for n in sem_result.get("nodes", []) + } | { + e.get("source_file", "") for e in sem_result.get("edges", []) + } + _sem_extracted.discard("") + _sem_types = {"document", "paper", "image"} + _manifest_files = { + ftype: [f for f in flist if ftype not in _sem_types or f in _sem_extracted] + for ftype, flist in files_by_type.items() + } + + if no_cluster: + # --no-cluster: dump the raw merged extraction as graph.json. + # No NetworkX, no community detection, no analysis sidecar. + # Dedupe nodes (by id) and parallel edges so the raw output matches the + # clustered path (whose DiGraph collapses both) and stays deterministic + # across modes (#1317; node dedup also collapses shared Swift module + # anchors emitted per importing file, #1327). + from graphify.build import dedupe_edges as _dedupe_edges, dedupe_nodes as _dedupe_nodes + from graphify.export import backup_if_protected as _backup + if ( + incremental_mode + and not code_files + and not semantic_files + and not deleted_files + and not pg_result.get("nodes") + and not pg_result.get("edges") + and not cargo_result.get("nodes") + and not cargo_result.get("edges") + ): + print( + "[graphify extract] no incremental changes detected " + "(--no-cluster); outputs left untouched." + ) + try: + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) + except Exception as exc: + print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) + stages.total() + sys.exit(0) + + merged["nodes"] = _dedupe_nodes(merged["nodes"]) + merged["edges"] = _dedupe_edges(merged["edges"]) + # Backfill source_file from endpoint nodes — this raw path bypasses + # build_from_json's backfill, and semantic edges sometimes omit it (#1279). + _node_sf = {n.get("id"): n.get("source_file") for n in merged["nodes"]} + for _e in merged["edges"]: + if not _e.get("source_file"): + _e["source_file"] = ( + _node_sf.get(_e.get("source")) or _node_sf.get(_e.get("target")) or "" + ) + _backup(graphify_out) + graph_json_path.write_text( + json.dumps(merged, indent=2), encoding="utf-8" + ) + stages.mark("write") + cost = _estimate_cost( + backend, merged["input_tokens"], merged["output_tokens"] + ) + print( + f"[graphify extract] wrote {graph_json_path} — " + f"{len(merged['nodes'])} nodes, {len(merged['edges'])} edges " + f"(no clustering)" + ) + if merged["input_tokens"] or merged["output_tokens"]: + print( + f"[graphify extract] tokens: " + f"{merged['input_tokens']:,} in / " + f"{merged['output_tokens']:,} out, " + f"est. cost: ${cost:.4f}" + ) + try: + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) + except Exception as exc: + print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) + if global_merge: + from graphify.global_graph import global_add as _global_add + _tag = global_repo_tag or target.name + try: + result = _global_add(graphify_out / "graph.json", _tag) + if result["skipped"]: + print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") + else: + print(f"[graphify global] '{_tag}' merged into global graph " + f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") + except Exception as exc: + print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) + stages.total() + sys.exit(0) + + # Build graph + cluster + score + write. + from graphify.build import ( + build as _build, + build_from_json as _build_from_json, + build_merge as _build_merge, + ) + from graphify.cluster import cluster as _cluster, score_all as _score_all + from graphify.export import to_json as _to_json + from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising + dedup_backend = backend if dedup_llm else None + if incremental_mode: + G = _build_merge( + [merged], + graph_path=existing_graph_path, + prune_sources=deleted_files or None, + dedup=True, + dedup_llm_backend=dedup_backend, + root=target, + ) + else: + G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend, root=target) + stages.mark("build") + if G.number_of_nodes() == 0: + print( + "[graphify extract] graph is empty — extraction produced no nodes. " + "Possible causes: all files skipped, binary-only corpus, or LLM " + "returned no edges.", + file=sys.stderr, + ) + sys.exit(1) + + communities = _cluster(G, resolution=cli_resolution, exclude_hubs_percentile=cli_exclude_hubs) + stages.mark("cluster") + cohesion = _score_all(G, communities) + try: + gods = _god_nodes(G) + except Exception: + gods = [] + try: + surprises = _surprising(G, communities) + except Exception: + surprises = [] + stages.mark("analyze") + + from graphify.export import backup_if_protected as _backup + _backup(graphify_out) + _to_json(G, communities, str(graph_json_path), force=True) + stages.mark("export") + if merged.get("output_tokens", 0) > 0: + (graphify_out / ".graphify_semantic_marker").write_text( + json.dumps({"output_tokens": merged["output_tokens"]}), encoding="utf-8" + ) + if global_merge: + from graphify.global_graph import global_add as _global_add + _tag = global_repo_tag or target.name + try: + result = _global_add(graphify_out / "graph.json", _tag) + if result["skipped"]: + print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") + else: + print(f"[graphify global] '{_tag}' merged into global graph " + f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") + except Exception as exc: + print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) + analysis = { + "communities": {str(k): v for k, v in communities.items()}, + "cohesion": {str(k): v for k, v in cohesion.items()}, + "gods": gods, + "surprises": surprises, + "tokens": { + "input": merged["input_tokens"], + "output": merged["output_tokens"], + }, + } + analysis_path.write_text(json.dumps(analysis, indent=2), encoding="utf-8") + try: + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) + except Exception as exc: + print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) + + cost = _estimate_cost(backend, merged["input_tokens"], merged["output_tokens"]) + print( + f"[graphify extract] wrote {graph_json_path}: " + f"{G.number_of_nodes()} nodes, {G.number_of_edges()} edges, " + f"{len(communities)} communities" + ) + print(f"[graphify extract] wrote {analysis_path}") + if incremental_mode: + print( + f"[graphify extract] incremental summary: " + f"{sem_cache_hits + unchanged_total} files cached/unchanged, " + f"{len(code_files) + sem_cache_misses} re-extracted, " + f"{len(deleted_files)} deleted" + ) + elif sem_cache_hits: + print(f"[graphify extract] semantic cache: {sem_cache_hits} cached, {sem_cache_misses} re-extracted") + if merged["input_tokens"] or merged["output_tokens"]: + print( + f"[graphify extract] tokens: " + f"{merged['input_tokens']:,} in / " + f"{merged['output_tokens']:,} out, " + f"est. cost (~{backend}): ${cost:.4f}" + ) + # extract intentionally stops at graph.json + analysis; the report and + # community labels are produced by `cluster-only` (or an agent's Step 5). + # Point standalone users at it so communities get named (#1097). + print( + "[graphify extract] next: run " + f"`graphify cluster-only {graphify_out.parent}` " + "to generate GRAPH_REPORT.md and name communities" + ) + stages.total() + + elif cmd == "cache-check": + # graphify cache-check [--root ] + # Reads file paths (one per line) from , checks semantic cache. + # Writes: + # graphify-out/.graphify_cached.json — already-cached nodes/edges/hyperedges + # graphify-out/.graphify_uncached.txt — paths that need extraction + # Stdout: "Cache: N hit, M miss" + from graphify.cache import check_semantic_cache + if len(sys.argv) < 3: + print("Usage: graphify cache-check [--root ]", file=sys.stderr) + sys.exit(1) + files_from = Path(sys.argv[2]) + root = Path(".") + i = 3 + while i < len(sys.argv): + if sys.argv[i] == "--root" and i + 1 < len(sys.argv): + root = Path(sys.argv[i + 1]) + i += 2 + else: + i += 1 + files = [f for f in files_from.read_text(encoding="utf-8").splitlines() if f.strip()] + cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(files, root) + out = root / _GRAPHIFY_OUT + out.mkdir(parents=True, exist_ok=True) + if cached_nodes or cached_edges or cached_hyperedges: + (out / ".graphify_cached.json").write_text( + json.dumps({"nodes": cached_nodes, "edges": cached_edges, "hyperedges": cached_hyperedges}, + ensure_ascii=False), + encoding="utf-8", + ) + (out / ".graphify_uncached.txt").write_text("\n".join(uncached), encoding="utf-8") + print(f"Cache: {len(files) - len(uncached)} hit, {len(uncached)} miss") + + elif cmd == "merge-chunks": + # graphify merge-chunks --out + # Concatenates .graphify_chunk_*.json files written by semantic subagents. + # Deduplicates nodes by id (first writer wins). Sums token counts. + import glob as _glob + if len(sys.argv) < 3: + print("Usage: graphify merge-chunks --out ", file=sys.stderr) + sys.exit(1) + out_path: Path | None = None + chunk_args: list[str] = [] + i = 2 + while i < len(sys.argv): + if sys.argv[i] == "--out" and i + 1 < len(sys.argv): + out_path = Path(sys.argv[i + 1]) + i += 2 + else: + chunk_args.append(sys.argv[i]) + i += 1 + if not out_path: + print("error: --out required", file=sys.stderr) + sys.exit(1) + chunk_files: list[str] = [] + for arg in chunk_args: + expanded = _glob.glob(arg) + chunk_files.extend(sorted(expanded) if expanded else [arg]) + merged: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} + seen_ids: set[str] = set() + for cf in chunk_files: + try: + chunk = json.loads(Path(cf).read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + print(f"[graphify merge-chunks] warning: skipping {cf}: {exc}", file=sys.stderr) + continue + for n in chunk.get("nodes", []): + if n.get("id") not in seen_ids: + seen_ids.add(n["id"]) + merged["nodes"].append(n) + merged["edges"].extend(chunk.get("edges", [])) + merged["hyperedges"].extend(chunk.get("hyperedges", [])) + merged["input_tokens"] += chunk.get("input_tokens", 0) + merged["output_tokens"] += chunk.get("output_tokens", 0) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(merged, ensure_ascii=False), encoding="utf-8") + print( + f"Merged {len(chunk_files)} chunks: {len(merged['nodes'])} nodes, {len(merged['edges'])} edges, " + f"{merged['input_tokens']:,} in / {merged['output_tokens']:,} out tokens" + ) + + elif cmd == "merge-semantic": + # graphify merge-semantic --cached --new --out + # Merges cached semantic results with freshly-extracted chunk results. + # Deduplicates nodes by id (cached entries take priority over new ones). + if len(sys.argv) < 3: + print("Usage: graphify merge-semantic --cached --new --out ", file=sys.stderr) + sys.exit(1) + cached_path: Path | None = None + new_path: Path | None = None + out_path2: Path | None = None + i = 2 + while i < len(sys.argv): + if sys.argv[i] == "--cached" and i + 1 < len(sys.argv): + cached_path = Path(sys.argv[i + 1]); i += 2 + elif sys.argv[i] == "--new" and i + 1 < len(sys.argv): + new_path = Path(sys.argv[i + 1]); i += 2 + elif sys.argv[i] == "--out" and i + 1 < len(sys.argv): + out_path2 = Path(sys.argv[i + 1]); i += 2 + else: + i += 1 + if not out_path2: + print("error: --out required", file=sys.stderr) + sys.exit(1) + empty: dict = {"nodes": [], "edges": [], "hyperedges": []} + cached_data = json.loads(cached_path.read_text(encoding="utf-8")) if cached_path and cached_path.exists() else empty + new_data = json.loads(new_path.read_text(encoding="utf-8")) if new_path and new_path.exists() else empty + seen_ids2: set[str] = set() + all_nodes: list[dict] = [] + for n in cached_data.get("nodes", []) + new_data.get("nodes", []): + if n.get("id") not in seen_ids2: + seen_ids2.add(n["id"]) + all_nodes.append(n) + merged2 = { + "nodes": all_nodes, + "edges": cached_data.get("edges", []) + new_data.get("edges", []), + "hyperedges": cached_data.get("hyperedges", []) + new_data.get("hyperedges", []), + } + out_path2.parent.mkdir(parents=True, exist_ok=True) + out_path2.write_text(json.dumps(merged2, ensure_ascii=False), encoding="utf-8") + print(f"Merged: {len(merged2['nodes'])} nodes, {len(merged2['edges'])} edges") + + elif Path(cmd).exists() or cmd in (".", "..") or cmd.startswith(("./", "../", "/", "~")): + # User ran `graphify ` directly — treat as `graphify extract `. + # Common when following the PowerShell note in README (`graphify .`) or + # copy-pasting skill invocations without the leading slash. + sys.argv.insert(2, sys.argv[1]) + sys.argv[1] = "extract" + _reenter_main() + else: + print(f"error: unknown command '{cmd}'", file=sys.stderr) + print("Run 'graphify --help' for usage.", file=sys.stderr) + sys.exit(1)