From 1a54f16b8028892300bfdaa63fcfe4c4a03af05d Mon Sep 17 00:00:00 2001 From: Safi Date: Mon, 6 Apr 2026 15:26:58 +0100 Subject: [PATCH] feat: always-on hooks and README updates for all platforms --- .gitignore | 4 + CHANGELOG.md | 23 + README.md | 49 +- .../2026-04-06-v3-platform-compatibility.md | 581 -------- ...-04-06-v3-platform-compatibility-design.md | 92 -- graphify/__main__.py | 90 +- skills/graphify/skill.md | 1204 ----------------- tests/test_install.py | 74 + 8 files changed, 219 insertions(+), 1898 deletions(-) delete mode 100644 docs/superpowers/plans/2026-04-06-v3-platform-compatibility.md delete mode 100644 docs/superpowers/specs/2026-04-06-v3-platform-compatibility-design.md delete mode 100644 skills/graphify/skill.md diff --git a/.gitignore b/.gitignore index b6215f814..a524bf7a1 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,7 @@ build/ .graphify/ graphify-out/ .graphify_*.json +.graphify_python +.claude/ +skills/ +docs/superpowers/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 6193cb1f7..826e63ce5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) + +## 0.3.0 (2026-04-06) + +- Add: multi-platform support — Codex (`skill-codex.md`), OpenCode (`skill-opencode.md`), OpenClaw (`skill-claw.md`) +- Add: `graphify install --platform ` routes skill to correct config directory +- Add: `graphify codex install` / `opencode install` / `claw install` — writes AGENTS.md for always-on graph-first behaviour +- Add: `graphify claude uninstall` / `codex uninstall` / `opencode uninstall` / `claw uninstall` +- Add: MIT license +- Fix: `build()` was silently dropping hyperedges when merging multiple extractions +- Refactor: `extract.py` 2527 → 1588 lines — replaced 12 copy-pasted language extractors with `LanguageConfig` dataclass + `_extract_generic()` +- Docs: clustering is graph-topology-based (no embeddings) — explained in README +- Docs: all missing flags documented (`--cluster-only`, `--no-viz`, `--neo4j-push`, `query --dfs`, `query --budget`, `add --author`, `add --contributor`) + +## 0.2.2 (2026-04-06) + +- Add: `graphify claude install` — writes graphify section to local CLAUDE.md + PreToolUse hook in `.claude/settings.json` +- Add: `graphify claude uninstall` — removes section and hook +- Add: `graphify hook install` — installs post-commit and post-checkout git hooks (platform-agnostic) +- Add: `graphify hook uninstall` / `hook status` +- Add: `graphify benchmark` CLI command +- Fix: node deduplication documented at all three layers + ## 0.1.8 (2026-04-05) - Fix: follow-up questions now check for wiki first (graphify-out/wiki/index.md) before falling back to graph.json diff --git a/README.md b/README.md index 20b0116a9..32fd499f5 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,13 @@ graphify-out/ graphify runs in two passes. First, a deterministic AST pass extracts structure from code files (classes, functions, imports, call graphs, docstrings, rationale comments) with no LLM needed. Second, Claude subagents run in parallel over docs, papers, and images to extract concepts, relationships, and design rationale. The results are merged into a NetworkX graph, clustered with Leiden community detection, and exported as interactive HTML, queryable JSON, and a plain-language audit report. +**Clustering is graph-topology-based — no embeddings.** Leiden finds communities by edge density. The semantic similarity edges that Claude extracts (`semantically_similar_to`, marked INFERRED) are already in the graph, so they influence community detection directly. The graph structure is the similarity signal — no separate embedding step or vector database needed. + Every relationship is tagged `EXTRACTED` (found directly in source), `INFERRED` (reasonable inference, with a confidence score), or `AMBIGUOUS` (flagged for review). You always know what was found vs guessed. ## Install -**Requires:** [Claude Code](https://claude.ai/code) and Python 3.10+ +**Requires:** Python 3.10+ and one of: [Claude Code](https://claude.ai/code), [Codex](https://openai.com/codex), [OpenCode](https://opencode.ai), or [OpenClaw](https://openclaw.ai) ```bash pip install graphifyy && graphify install @@ -54,23 +56,22 @@ Then open your AI coding assistant and type: /graphify . ``` -### Make Claude always use the graph (recommended) +### Make your assistant always use the graph (recommended) After building a graph, run this once in your project: -```bash -graphify claude install -``` +| Platform | Command | +|----------|---------| +| Claude Code | `graphify claude install` | +| Codex | `graphify codex install` | +| OpenCode | `graphify opencode install` | +| OpenClaw | `graphify claw install` | -This does two things: +**Claude Code** does two things: writes a `CLAUDE.md` section telling Claude to read `graphify-out/GRAPH_REPORT.md` before answering architecture questions, and installs a **PreToolUse hook** (`settings.json`) that fires before every Glob and Grep call. If a knowledge graph exists, Claude sees: _"graphify: Knowledge graph exists. Read GRAPH_REPORT.md for god nodes and community structure before searching raw files."_ — so Claude navigates via the graph instead of grepping through every file. -1. **CLAUDE.md rules** - tells Claude to read `graphify-out/GRAPH_REPORT.md` before answering architecture questions, and to rebuild the graph after editing code files. +**Codex, OpenCode, OpenClaw** write the same rules to `AGENTS.md` in your project root. These platforms don't support PreToolUse hooks, so AGENTS.md is the always-on mechanism. -2. **PreToolUse hook** (`settings.json`) - fires automatically before every Glob and Grep call. If a knowledge graph exists, Claude sees: _"graphify: Knowledge graph exists. Read GRAPH_REPORT.md for god nodes and community structure before searching raw files."_ This means Claude navigates via the graph instead of grepping through every file - faster answers, fewer wasted tool calls, and responses grounded in the actual structure of your codebase rather than keyword matches. - -Without this, Claude will grep raw files by default even when a graph exists. With it, the graph becomes the first thing Claude reaches for. - -Uninstall with `graphify claude uninstall`. +Uninstall with the matching uninstall command (e.g. `graphify claude uninstall`).
Manual install (curl) @@ -97,12 +98,18 @@ When the user types `/graphify`, invoke the Skill tool with `skill: "graphify"` /graphify ./raw # run on a specific folder /graphify ./raw --mode deep # more aggressive INFERRED edge extraction /graphify ./raw --update # re-extract only changed files, merge into existing graph +/graphify ./raw --cluster-only # rerun clustering on existing graph, no re-extraction +/graphify ./raw --no-viz # skip HTML, just produce report + JSON /graphify ./raw --obsidian # also generate Obsidian vault (opt-in) /graphify add https://arxiv.org/abs/1706.03762 # fetch a paper, save, update graph /graphify add https://x.com/karpathy/status/... # fetch a tweet +/graphify add https://... --author "Name" # tag the original author +/graphify add https://... --contributor "Name" # tag who added it to the corpus /graphify query "what connects attention to the optimizer?" +/graphify query "what connects attention to the optimizer?" --dfs # trace a specific path +/graphify query "what connects attention to the optimizer?" --budget 1500 # cap at N tokens /graphify path "DigestAuth" "Response" /graphify explain "SwinTransformer" @@ -111,10 +118,20 @@ When the user types `/graphify`, invoke the Skill tool with `skill: "graphify"` /graphify ./raw --svg # export graph.svg /graphify ./raw --graphml # export graph.graphml (Gephi, yEd) /graphify ./raw --neo4j # generate cypher.txt for Neo4j +/graphify ./raw --neo4j-push bolt://localhost:7687 # push directly to a running Neo4j instance /graphify ./raw --mcp # start MCP stdio server -graphify hook install # git hooks - rebuilds graph on commit and branch switch -graphify claude install # always-on: CLAUDE.md + PreToolUse hook for this project +# git hooks - platform-agnostic, rebuild graph on commit and branch switch +graphify hook install +graphify hook uninstall +graphify hook status + +# always-on assistant instructions - platform-specific +graphify claude install # CLAUDE.md + PreToolUse hook (Claude Code) +graphify claude uninstall +graphify codex install # AGENTS.md (Codex) +graphify opencode install # AGENTS.md (OpenCode) +graphify claw install # AGENTS.md (OpenClaw) ``` Works with any mix of file types: @@ -162,11 +179,11 @@ Token reduction scales with corpus size. 6 files fits in a context window anyway ## Privacy -graphify sends file contents to the Claude API (Anthropic) for semantic extraction of docs, papers, and images. Code files are processed locally via tree-sitter AST — no file contents leave your machine for code. No telemetry, usage tracking, or analytics of any kind. The only network calls are to Anthropic's API during extraction, using your own API key via Claude Code. +graphify sends file contents to your AI coding assistant's underlying model API for semantic extraction of docs, papers, and images — Anthropic (Claude Code), OpenAI (Codex), or whichever provider your platform uses. Code files are processed locally via tree-sitter AST — no file contents leave your machine for code. No telemetry, usage tracking, or analytics of any kind. The only network calls are to your platform's model API during extraction, using your own API key. ## Tech stack -NetworkX + Leiden (graspologic) + tree-sitter + Claude + vis.js. No Neo4j required, no server, runs entirely locally. +NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Semantic extraction via Claude (Claude Code), GPT-4 (Codex), or whichever model your platform runs. No Neo4j required, no server, runs entirely locally.
Contributing diff --git a/docs/superpowers/plans/2026-04-06-v3-platform-compatibility.md b/docs/superpowers/plans/2026-04-06-v3-platform-compatibility.md deleted file mode 100644 index 2dc86fd5a..000000000 --- a/docs/superpowers/plans/2026-04-06-v3-platform-compatibility.md +++ /dev/null @@ -1,581 +0,0 @@ -# v3 Platform Compatibility Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add Codex, OpenCode, and OpenClaw platform support via platform-specific skill files and a `graphify install --platform X` flag. - -**Architecture:** The only section that differs between platforms is Step B2 (semantic extraction subagent dispatch) in skill.md. Three new skill files are created — one per platform — each identical to skill.md except for that one section. The `install()` function in `__main__.py` gains a `--platform` flag that copies the right skill file to the right config directory. - -**Tech Stack:** Python 3.10+, pathlib, shutil, argparse (no new deps) - ---- - -## File Map - -| File | Action | Purpose | -|------|--------|---------| -| `graphify/skill.md` | Read-only | Source of truth — unchanged | -| `graphify/skill-codex.md` | Create | Codex variant (spawn_agent + wait) | -| `graphify/skill-opencode.md` | Create | OpenCode variant (@mention dispatch) | -| `graphify/skill-claw.md` | Create | OpenClaw variant (sequential extraction) | -| `graphify/__main__.py` | Modify | Add --platform flag to install() and main() | -| `pyproject.toml` | Modify | Add 3 new skill files to package-data | -| `tests/test_install.py` | Create | Platform routing tests | -| `README.md` | Modify | Platform table + token efficiency clarification | - ---- - -## Task 1: Create the v3 branch - -**Files:** none (git only) - -- [ ] **Step 1: Create and switch to v3 branch** - -```bash -cd /home/safi/graphify -git checkout -b v3 -``` - -Expected: `Switched to a new branch 'v3'` - -- [ ] **Step 2: Verify branch** - -```bash -git branch --show-current -``` - -Expected: `v3` - ---- - -## Task 2: Create `skill-codex.md` - -skill-codex.md is identical to skill.md with one change: Step B2 replaces `Agent` tool calls with `spawn_agent` + `wait` + `close_agent` calls. - -**Files:** -- Create: `graphify/skill-codex.md` - -- [ ] **Step 1: Copy skill.md as the base** - -```bash -cp graphify/skill.md graphify/skill-codex.md -``` - -- [ ] **Step 2: Open `graphify/skill-codex.md` and replace the Step B2 section** - -Find this block (starts at "**Step B2 - Dispatch ALL subagents in a single message**", ends before "**Step B3**"): - -Replace the entire Step B2 section with: - -```markdown -**Step B2 - Dispatch ALL subagents in a single message (Codex)** - -> **Codex platform:** This step uses `spawn_agent` + `wait` + `close_agent` instead of the Agent tool. -> Requires `multi_agent = true` in `~/.codex/config.toml`. If you get an error about multi-agent support, ask the user to add that config line and restart Codex. - -Call `spawn_agent` once per chunk — all in the same response so they run in parallel: - -``` -spawn_agent(agent_type="worker", message="Your task is to perform the following. Follow the instructions below exactly.\n\n\nYou are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment.\nOutput ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble.\n\nFiles (chunk CHUNK_NUM of TOTAL_CHUNKS):\nFILE_LIST\n\n[copy the extraction rules and JSON schema verbatim from the existing Step B2 content — it's already in the file from the cp step]\n\n\nExecute this now. Output ONLY the structured JSON response.") -``` - -Collect all handles. Then for each handle: -``` -result = wait(handle) -close_agent(handle) -``` - -Parse each result as JSON. Accumulate nodes/edges/hyperedges across all results into `.graphify_semantic_new.json`. - -If `spawn_agent` is not available, tell the user: "Codex multi-agent support is not enabled. Add `multi_agent = true` under `[features]` in `~/.codex/config.toml` and restart Codex." -``` - -- [ ] **Step 3: Verify the file looks correct** - -```bash -grep -n "spawn_agent\|Step B2\|Step B3" graphify/skill-codex.md | head -20 -``` - -Expected: lines showing spawn_agent in B2 and Step B3 after it. - -- [ ] **Step 4: Commit** - -```bash -git add graphify/skill-codex.md -git commit -m "add skill-codex.md for Codex platform (spawn_agent parallel extraction)" -``` - ---- - -## Task 3: Create `skill-opencode.md` - -**Files:** -- Create: `graphify/skill-opencode.md` - -- [ ] **Step 1: Copy skill.md as the base** - -```bash -cp graphify/skill.md graphify/skill-opencode.md -``` - -- [ ] **Step 2: Open `graphify/skill-opencode.md` and replace the Step B2 section** - -Replace the entire Step B2 section with: - -```markdown -**Step B2 - Dispatch ALL subagents in a single message (OpenCode)** - -> **OpenCode platform:** This step uses OpenCode's `@mention` dispatch instead of the Agent tool. - -Dispatch all chunks in a single response. Each `@mention` runs in parallel: - -``` -@agent Chunk CHUNK_NUM of TOTAL_CHUNKS: You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. Output ONLY valid JSON matching the schema below. - -Files: -FILE_LIST - -[copy the extraction rules and JSON schema verbatim from the existing Step B2 content — already in the file from the cp step] -``` - -One `@mention` block per chunk. All in the same message — this is what makes them parallel. - -Wait for all agents to return. Parse each response as JSON. Accumulate nodes/edges/hyperedges across all results into `.graphify_semantic_new.json`. -``` - -- [ ] **Step 3: Verify the file looks correct** - -```bash -grep -n "@mention\|Step B2\|Step B3" graphify/skill-opencode.md | head -20 -``` - -Expected: lines showing @mention in B2 and Step B3 after it. - -- [ ] **Step 4: Commit** - -```bash -git add graphify/skill-opencode.md -git commit -m "add skill-opencode.md for OpenCode platform (@mention parallel extraction)" -``` - ---- - -## Task 4: Create `skill-claw.md` - -OpenClaw's agent support is MVP/incomplete so extraction is sequential — the orchestrating LLM reads each file and extracts directly. - -**Files:** -- Create: `graphify/skill-claw.md` - -- [ ] **Step 1: Copy skill.md as the base** - -```bash -cp graphify/skill.md graphify/skill-claw.md -``` - -- [ ] **Step 2: Open `graphify/skill-claw.md` and replace the Step B2 section** - -Replace the entire Step B2 section with: - -```markdown -**Step B2 - Sequential extraction (OpenClaw)** - -> **OpenClaw platform:** OpenClaw's multi-agent support is still early. Extraction runs sequentially — you read each file yourself and extract directly. This is slower than parallel platforms but reliable. - -Load files from `.graphify_uncached.txt`. For each file, one at a time: - -1. Read the file contents -2. Extract nodes, edges, and hyperedges following the same rules and schema as the parallel variant (see schema below) -3. Accumulate results into a running JSON object - -Apply all the same extraction rules: -- EXTRACTED / INFERRED / AMBIGUOUS confidence with confidence_score on every edge -- rationale_for nodes for design decisions and WHY comments -- semantically_similar_to edges for cross-file conceptual links (non-obvious only) -- hyperedges for groups of 3+ nodes (max 3 per file) -- DEEP_MODE: more aggressive INFERRED edges if --mode deep was given - -Schema (same as parallel variant): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} - -After processing all files, write the accumulated result to `.graphify_semantic_new.json`. -``` - -- [ ] **Step 3: Also remove the timing estimate block from Step B** - -In skill-claw.md, find and remove this paragraph (it only applies to parallel dispatch): - -``` -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" -``` - -Replace with: - -``` -Print: "Semantic extraction: N files (sequential — OpenClaw platform)" -``` - -- [ ] **Step 4: Verify** - -```bash -grep -n "sequential\|Step B2\|Step B3\|spawn_agent\|@mention" graphify/skill-claw.md | head -20 -``` - -Expected: "sequential" appears in B2, no spawn_agent or @mention. - -- [ ] **Step 5: Commit** - -```bash -git add graphify/skill-claw.md -git commit -m "add skill-claw.md for OpenClaw platform (sequential extraction)" -``` - ---- - -## Task 5: Update `pyproject.toml` package-data - -**Files:** -- Modify: `pyproject.toml` - -- [ ] **Step 1: Update package-data to include the three new skill files** - -In `pyproject.toml`, find: - -```toml -[tool.setuptools.package-data] -graphify = ["skill.md"] -``` - -Replace with: - -```toml -[tool.setuptools.package-data] -graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-claw.md"] -``` - -- [ ] **Step 2: Verify** - -```bash -grep -A2 "package-data" pyproject.toml -``` - -Expected: all four skill files listed. - -- [ ] **Step 3: Commit** - -```bash -git add pyproject.toml -git commit -m "include platform skill files in package-data" -``` - ---- - -## Task 6: Add `--platform` flag to install command - -**Files:** -- Modify: `graphify/__main__.py` - -- [ ] **Step 1: Write the failing test first** - -Create `tests/test_install.py`: - -```python -"""Tests for graphify install --platform routing.""" -import shutil -from pathlib import Path -import pytest -from unittest.mock import patch - - -PLATFORMS = { - "claude": ("skill.md", ".claude/skills/graphify/SKILL.md"), - "codex": ("skill-codex.md", ".agents/skills/graphify/SKILL.md"), - "opencode": ("skill-opencode.md", ".config/opencode/skills/graphify/SKILL.md"), - "claw": ("skill-claw.md", ".claw/skills/graphify/SKILL.md"), -} - - -def test_install_default_uses_claude_skill(tmp_path): - """install() with no platform copies skill.md to ~/.claude/skills/graphify/SKILL.md""" - from graphify.__main__ import install - with patch("graphify.__main__.Path.home", return_value=tmp_path): - install(platform="claude") - dst = tmp_path / ".claude" / "skills" / "graphify" / "SKILL.md" - assert dst.exists() - - -def test_install_codex_copies_correct_file(tmp_path): - from graphify.__main__ import install - with patch("graphify.__main__.Path.home", return_value=tmp_path): - install(platform="codex") - dst = tmp_path / ".agents" / "skills" / "graphify" / "SKILL.md" - assert dst.exists() - - -def test_install_opencode_copies_correct_file(tmp_path): - from graphify.__main__ import install - with patch("graphify.__main__.Path.home", return_value=tmp_path): - install(platform="opencode") - dst = tmp_path / ".config" / "opencode" / "skills" / "graphify" / "SKILL.md" - assert dst.exists() - - -def test_install_claw_copies_correct_file(tmp_path): - from graphify.__main__ import install - with patch("graphify.__main__.Path.home", return_value=tmp_path): - install(platform="claw") - dst = tmp_path / ".claw" / "skills" / "graphify" / "SKILL.md" - assert dst.exists() - - -def test_install_unknown_platform_exits(tmp_path): - from graphify.__main__ import install - with patch("graphify.__main__.Path.home", return_value=tmp_path): - with pytest.raises(SystemExit): - install(platform="unknown") - - -def test_all_skill_files_exist_in_package(): - """Verify all platform skill files are present in the installed package.""" - import graphify - pkg_dir = Path(graphify.__file__).parent - for src_name, _ in PLATFORMS.values(): - skill_path = pkg_dir / src_name - assert skill_path.exists(), f"Missing skill file: {src_name}" -``` - -- [ ] **Step 2: Run the test to verify it fails** - -```bash -python -m pytest tests/test_install.py -v --tb=short 2>&1 | head -40 -``` - -Expected: FAIL — `install()` doesn't accept a `platform` argument yet. - -- [ ] **Step 3: Update `install()` in `graphify/__main__.py`** - -Replace the current `install()` function and add `_PLATFORM_CONFIG`: - -```python -_PLATFORM_CONFIG = { - "claude": { - "skill_file": "skill.md", - "skill_dst": Path(".claude") / "skills" / "graphify" / "SKILL.md", - "claude_md": True, # only Claude Code gets CLAUDE.md registration - }, - "codex": { - "skill_file": "skill-codex.md", - "skill_dst": Path(".agents") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - }, - "opencode": { - "skill_file": "skill-opencode.md", - "skill_dst": Path(".config") / "opencode" / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - }, - "claw": { - "skill_file": "skill-claw.md", - "skill_dst": Path(".claw") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - }, -} - - -def install(platform: str = "claude") -> None: - if platform not in _PLATFORM_CONFIG: - print(f"error: unknown platform '{platform}'. Choose from: {', '.join(_PLATFORM_CONFIG)}", file=sys.stderr) - sys.exit(1) - - cfg = _PLATFORM_CONFIG[platform] - skill_src = Path(__file__).parent / cfg["skill_file"] - if not skill_src.exists(): - print(f"error: {cfg['skill_file']} not found in package - reinstall graphify", file=sys.stderr) - sys.exit(1) - - skill_dst = Path.home() / cfg["skill_dst"] - skill_dst.parent.mkdir(parents=True, exist_ok=True) - shutil.copy(skill_src, skill_dst) - print(f" skill installed → {skill_dst}") - - if cfg["claude_md"]: - # Register in ~/.claude/CLAUDE.md (Claude Code only) - claude_md = Path.home() / ".claude" / "CLAUDE.md" - if claude_md.exists(): - content = claude_md.read_text() - if "graphify" in content: - print(f" CLAUDE.md → already registered (no change)") - else: - claude_md.write_text(content.rstrip() + _SKILL_REGISTRATION) - print(f" CLAUDE.md → skill registered in {claude_md}") - else: - claude_md.parent.mkdir(parents=True, exist_ok=True) - claude_md.write_text(_SKILL_REGISTRATION.lstrip()) - print(f" CLAUDE.md → created at {claude_md}") - - print() - print("Done. Open your AI coding assistant and type:") - print() - print(" /graphify .") - print() -``` - -- [ ] **Step 4: Update `main()` to pass `--platform` to `install()`** - -In `main()`, find the `if cmd == "install":` block: - -```python - if cmd == "install": - install() -``` - -Replace with: - -```python - if cmd == "install": - platform = "claude" - args = sys.argv[2:] - i = 0 - while i < len(args): - if args[i].startswith("--platform="): - platform = args[i].split("=", 1)[1] - i += 1 - elif args[i] == "--platform" and i + 1 < len(args): - platform = args[i + 1] - i += 2 - else: - i += 1 - install(platform=platform) -``` - -- [ ] **Step 5: Update the help text in `main()`** - -Find: -```python - print(" install copy skill to ~/.claude/skills/ and register in CLAUDE.md") -``` - -Replace with: -```python - print(" install [--platform P] copy skill to platform config dir (claude|codex|opencode|claw)") -``` - -- [ ] **Step 6: Run tests to verify they pass** - -```bash -python -m pytest tests/test_install.py -v --tb=short -``` - -Expected: all 6 tests PASS. - -- [ ] **Step 7: Run the full test suite to check for regressions** - -```bash -python -m pytest tests/ -q --tb=short 2>&1 | tail -20 -``` - -Expected: existing tests still pass. - -- [ ] **Step 8: Commit** - -```bash -git add graphify/__main__.py tests/test_install.py -git commit -m "add --platform flag to graphify install (codex, opencode, claw)" -``` - ---- - -## Task 7: Update README - -**Files:** -- Modify: `README.md` - -- [ ] **Step 1: Add platform support table under the Install section** - -After the `pip install graphifyy && graphify install` code block, add: - -```markdown -### Platform support - -| Platform | Install command | -|----------|----------------| -| Claude Code | `graphify install` | -| Codex | `graphify install --platform codex` | -| OpenCode | `graphify install --platform opencode` | -| OpenClaw | `graphify install --platform claw` | - -Codex users also need `multi_agent = true` under `[features]` in `~/.codex/config.toml` for parallel extraction. OpenClaw uses sequential extraction (parallel agent support is still early on that platform). -``` - -- [ ] **Step 2: Clarify token efficiency — find the benchmark section** - -Find the line: -``` -**Token benchmark** - printed automatically after every run. On a mixed corpus (Karpathy repos + papers + images): **71.5x** fewer tokens per query vs reading raw files. -``` - -Replace with: -``` -**Token benchmark** - printed automatically after every run. On a mixed corpus (Karpathy repos + papers + images): **71.5x** fewer tokens per query vs reading raw files. The first run extracts and builds the graph (this costs tokens). Every subsequent query reads the compact graph instead of raw files — that's where the savings compound. The SHA256 cache means re-runs only re-process changed files. -``` - -- [ ] **Step 3: Verify README renders correctly** - -```bash -grep -n "Platform support\|multi_agent\|first run extracts" README.md -``` - -Expected: all three lines found. - -- [ ] **Step 4: Commit** - -```bash -git add README.md -git commit -m "add platform support table and clarify token efficiency in README" -``` - ---- - -## Task 8: Final verification - -- [ ] **Step 1: Run the full test suite** - -```bash -python -m pytest tests/ -q --tb=short 2>&1 | tail -20 -``` - -Expected: all tests pass, no regressions. - -- [ ] **Step 2: Verify all four skill files are present in the package** - -```bash -ls graphify/skill*.md -``` - -Expected: -``` -graphify/skill.md -graphify/skill-codex.md -graphify/skill-opencode.md -graphify/skill-claw.md -``` - -- [ ] **Step 3: Smoke test each install path** - -```bash -python -m graphify.__main__ install --platform codex 2>&1 | head -5 -python -m graphify.__main__ install --platform opencode 2>&1 | head -5 -python -m graphify.__main__ install --platform claw 2>&1 | head -5 -python -m graphify.__main__ install --platform unknown 2>&1 -``` - -Expected: first three print "skill installed →", last prints "error: unknown platform". - -- [ ] **Step 4: Push v3 branch** - -```bash -git push -u origin v3 -``` diff --git a/docs/superpowers/specs/2026-04-06-v3-platform-compatibility-design.md b/docs/superpowers/specs/2026-04-06-v3-platform-compatibility-design.md deleted file mode 100644 index d04b86bbf..000000000 --- a/docs/superpowers/specs/2026-04-06-v3-platform-compatibility-design.md +++ /dev/null @@ -1,92 +0,0 @@ -# v3 Platform Compatibility Design - -**Date:** 2026-04-06 -**Status:** Approved - -## Problem - -graphify's `skill.md` uses the Claude Code `Agent` tool for parallel semantic extraction. Users on Codex, OpenCode, and OpenClaw cannot use the skill. v3 adds platform-specific skill files so graphify works natively on all four platforms. - -## Scope - -- Four platform-specific skill files (one already exists) -- `graphify install --platform X` routing -- README clarifications (token efficiency, platform table) -- No always-on project hooks for non-Claude-Code platforms in v3 (deferred to v3.1) - -## What Changes Per Platform - -The semantic extraction step (Step 3B) is the **only** section that differs. AST extraction, merging, clustering, labelling, export, and benchmarking are identical across all platforms and live in the shared Python CLI. - -| Platform | Extraction approach | Rationale | -|----------|-------------------|-----------| -| Claude Code | Parallel `Agent` tool calls | Current behavior, unchanged | -| Codex | Parallel `spawn_agent` + `wait` + `close_agent` | Codex multi-agent API (`multi_agent = true` required) | -| OpenCode | Parallel `@mention` dispatches | OpenCode's native subagent system | -| OpenClaw | Sequential loop — orchestrator extracts each file itself | OpenClaw agent support is MVP/incomplete; sequential is reliable | - -## File Structure - -``` -graphify/ -├── skill.md # Claude Code (unchanged) -├── skill-codex.md # Codex — parallel via spawn_agent -├── skill-opencode.md # OpenCode — parallel via @mention -├── skill-claw.md # OpenClaw — sequential extraction -``` - -All four files ship in the PyPI package via `pyproject.toml` `package-data`. - -## Install Command - -`graphify install` gains a `--platform` flag: - -``` -graphify install # Claude Code → ~/.claude/skills/graphify/SKILL.md -graphify install --platform codex # Codex → ~/.agents/skills/graphify/SKILL.md -graphify install --platform opencode # OpenCode → ~/.config/opencode/skills/graphify/SKILL.md -graphify install --platform claw # OpenClaw → ~/.claw/skills/graphify/SKILL.md -``` - -Behaviour: -- Creates target directory if it doesn't exist (same as current Claude Code install) -- If the platform's root config directory doesn't exist, prints a warning and exits cleanly: `"Codex config directory not found — is Codex installed?"` -- `--platform` is optional; default is `claude` (current behaviour preserved) - -## Skill File Content - -Each file follows the same structure as `skill.md`. The extraction step (Step 3B) is rewritten for the platform: - -**Codex (`skill-codex.md`):** -- For each uncached file, call `spawn_agent(agent_type="worker", message=)` -- Collect all agent handles, call `wait()` on each, then `close_agent()` -- Requires user to have `multi_agent = true` in `~/.codex/config.toml`; skill notes this requirement - -**OpenCode (`skill-opencode.md`):** -- For each uncached file, dispatch via `@mention` with the extraction prompt -- Collect results as agents complete - -**OpenClaw (`skill-claw.md`):** -- Loop over uncached files sequentially -- Orchestrating LLM reads each file and extracts concepts/relationships/edges directly -- Slower than parallel platforms but reliable given OpenClaw's MVP agent status -- A note in the skill explains why: "OpenClaw's multi-agent support is still early; sequential extraction ensures reliability" - -## README Changes - -1. Add "Platform support" table under the Install section -2. Clarify token efficiency: *"First run extracts and builds the graph — subsequent queries read the compact graph instead of raw files. The 71.5x reduction applies per query, and the cache means re-runs only re-process changed files."* -3. Note sequential extraction on OpenClaw with brief explanation - -## Not In Scope (v3) - -- `graphify codex install` / `graphify opencode install` (always-on project hooks for non-CC platforms) — deferred to v3.1 -- Gemini CLI support — not enough information yet -- Copilot CLI support — not enough information yet - -## Testing - -- Unit tests in `tests/test_install.py`: verify `--platform X` routes to correct source file and target path -- Package data test: assert all four skill files are present in the installed package -- No execution tests for platform-specific extraction (requires live platform) -- Evals before release: run each platform skill on a real corpus, verify graph output is equivalent diff --git a/graphify/__main__.py b/graphify/__main__.py index cb81d39b5..951b661f2 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -107,6 +107,71 @@ Rules: _CLAUDE_MD_MARKER = "## graphify" +# AGENTS.md section for Codex, OpenCode, and OpenClaw. +# All three platforms read AGENTS.md in the project root for persistent instructions. +_AGENTS_MD_SECTION = """\ +## graphify + +This project has a graphify knowledge graph at graphify-out/. + +Rules: +- Before answering architecture or codebase questions, read graphify-out/GRAPH_REPORT.md for god nodes and community structure +- If graphify-out/wiki/index.md exists, navigate it instead of reading raw files +- After modifying code files in this session, run `python3 -c "from graphify.watch import _rebuild_code; from pathlib import Path; _rebuild_code(Path('.'))"` to keep the graph current +""" + +_AGENTS_MD_MARKER = "## graphify" + + +def _agents_install(project_dir: Path, platform: str) -> None: + """Write the graphify section to the local AGENTS.md (Codex/OpenCode/OpenClaw).""" + target = (project_dir or Path(".")) / "AGENTS.md" + + if target.exists(): + content = target.read_text() + if _AGENTS_MD_MARKER in content: + print(f"graphify already configured in AGENTS.md") + return + new_content = content.rstrip() + "\n\n" + _AGENTS_MD_SECTION + else: + new_content = _AGENTS_MD_SECTION + + target.write_text(new_content) + print(f"graphify section written to {target.resolve()}") + print() + print(f"{platform.capitalize()} will now check the knowledge graph before answering") + print("codebase questions and rebuild it after code changes.") + print() + print("Note: unlike Claude Code, there is no PreToolUse hook equivalent for") + print(f"{platform.capitalize()} — the AGENTS.md rules are the always-on mechanism.") + + +def _agents_uninstall(project_dir: Path) -> None: + """Remove the graphify section from the local AGENTS.md.""" + target = (project_dir or Path(".")) / "AGENTS.md" + + if not target.exists(): + print("No AGENTS.md found in current directory - nothing to do") + return + + content = target.read_text() + if _AGENTS_MD_MARKER not in content: + print("graphify section not found in AGENTS.md - nothing to do") + return + + cleaned = re.sub( + r"\n*## graphify\n.*?(?=\n## |\Z)", + "", + content, + flags=re.DOTALL, + ).rstrip() + if cleaned: + target.write_text(cleaned + "\n") + print(f"graphify section removed from {target.resolve()}") + else: + target.unlink() + print(f"AGENTS.md was empty after removal - deleted {target.resolve()}") + def claude_install(project_dir: Path | None = None) -> None: """Write the graphify section to the local CLAUDE.md.""" @@ -213,11 +278,17 @@ def main() -> None: print("Commands:") print(" install [--platform P] copy skill to platform config dir (claude|codex|opencode|claw)") print(" benchmark [graph.json] measure token reduction vs naive full-corpus approach") - print(" hook install install post-commit git hook (auto-rebuilds graph on commit)") - print(" hook uninstall remove post-commit git hook") - print(" hook status check if hook is installed") - print(" claude install write graphify section to local CLAUDE.md") - print(" claude uninstall remove graphify section from local CLAUDE.md") + print(" hook install install post-commit/post-checkout git hooks (all platforms)") + print(" hook uninstall remove git hooks") + print(" hook status check if git hooks are installed") + print(" claude install write graphify section to CLAUDE.md + PreToolUse hook (Claude Code)") + print(" claude uninstall remove graphify section from CLAUDE.md + PreToolUse hook") + print(" codex install write graphify section to AGENTS.md (Codex)") + print(" codex uninstall remove graphify section from AGENTS.md") + print(" opencode install write graphify section to AGENTS.md (OpenCode)") + print(" opencode uninstall remove graphify section from AGENTS.md") + print(" claw install write graphify section to AGENTS.md (OpenClaw)") + print(" claw uninstall remove graphify section from AGENTS.md") print() return @@ -245,6 +316,15 @@ def main() -> None: else: print("Usage: graphify claude [install|uninstall]", file=sys.stderr) sys.exit(1) + elif cmd in ("codex", "opencode", "claw"): + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd == "install": + _agents_install(Path("."), cmd) + elif subcmd == "uninstall": + _agents_uninstall(Path(".")) + else: + print(f"Usage: graphify {cmd} [install|uninstall]", file=sys.stderr) + sys.exit(1) elif cmd == "hook": from graphify.hooks import install as hook_install, uninstall as hook_uninstall, status as hook_status subcmd = sys.argv[2] if len(sys.argv) > 2 else "" diff --git a/skills/graphify/skill.md b/skills/graphify/skill.md deleted file mode 100644 index 444f080f1..000000000 --- a/skills/graphify/skill.md +++ /dev/null @@ -1,1204 +0,0 @@ ---- -name: graphify -description: any input (code, docs, papers, images) → knowledge graph → clustered communities → HTML + JSON + audit report -trigger: /graphify ---- - -# /graphify - -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. - -## Usage - -``` -/graphify # full pipeline on current directory → Obsidian vault -/graphify # full pipeline on specific path -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node -``` - -## What graphify is for - -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that Claude alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes → one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) - -## What You Must Do When Invoked - -If no path was given, use `.` (current directory). Do not ask the user for a path. - -Follow these steps in order. Do not skip steps. - -### Step 1 - Ensure graphify is installed - -```bash -python3 -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 -``` - -If the import succeeds, print nothing and move straight to Step 2. - -### Step 2 - Detect files - -```bash -python3 -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) -" > .graphify_detect.json -``` - -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: - -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files -``` - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 3 - no need to ask anything. - -### Step 3 - Extract entities and relationships - -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (Claude, costs tokens). - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. - -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -python3 -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('.graphify_detect.json').read_text()) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files) - Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) - print('No code files - skipping AST extraction') -" -``` - -#### Part B - Semantic extraction (parallel subagents) - -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -```bash -python3 -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('.graphify_detect.json').read_text()) -all_files = [f for files in detect['files'].values() for f in files] - -cached_nodes, cached_edges, uncached = check_semantic_cache(all_files) - -if cached_nodes or cached_edges: - Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges})) -Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). - -**Step B2 - Dispatch ALL subagents in a single message** - -Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. - -Concrete example for 3 chunks: -``` -[Agent tool call 1: files 1-15] -[Agent tool call 2: files 16-30] -[Agent tool call 3: files 31-45] -``` -All three in one message. Not three separate messages. - -Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE): - -``` -You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. -Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. - -Files (chunk CHUNK_NUM of TOTAL_CHUNKS): -FILE_LIST - -Rules: -- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") -- INFERRED: reasonable inference (shared data structure, implied dependency) -- AMBIGUOUS: uncertain - flag for review, do not omit - -Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). - Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. -Image files: use vision to understand what the image IS - do not just OCR. - UI screenshot: layout patterns, design decisions, key elements, purpose. - Chart: metric, trend/insight, data source. - Tweet/post: claim as node, author, concepts mentioned. - Diagram: components and connections. - Research figure: what it demonstrates, method, result. - Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. - -DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, - shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - -Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: -- Two functions that both validate user input but never call each other -- A class in code and a concept in a paper that describe the same algorithm -- Two error types that handle the same failure mode differently -Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. - -Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: -- All classes that implement a common protocol or interface -- All functions in an authentication flow (even if they don't all call each other) -- All concepts from a paper section that form one coherent idea -Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. - -If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, - contributor onto every node from that file. - -confidence_score rules: -- EXTRACTED edges: confidence_score must be 1.0 -- INFERRED edges: score 0.4-0.9 based on how certain you are. - Strong structural inference (e.g. two classes clearly share data): 0.8-0.9. - Reasonable but not certain: 0.6-0.7. Weak inference: 0.4-0.5. -- AMBIGUOUS edges: score 0.1-0.3 - -Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} -``` - -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- If a subagent returned valid JSON with `nodes` and `edges`, include it and save each file's nodes/edges to the cache -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed, stop and tell the user. - -Save new results to cache: -```bash -python3 -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', [])) -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `.graphify_semantic.json`: -```bash -python3 -c " -import json -from pathlib import Path - -cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[]} -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` - -#### Part C - Merge AST + semantic into final extraction - -```bash -python3 -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('.graphify_ast.json').read_text()) -sem = json.loads(Path('.graphify_semantic.json').read_text()) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` - -### Step 4 - Build graph, cluster, analyze, generate outputs - -```bash -mkdir -p graphify-out -python3 -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) - -G = build_from_json(extraction) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" -``` - -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. - -### Step 5 - Label communities - -Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - -```bash -python3 -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) -print('Report updated with community labels') -" -``` - -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. - -### Step 6 - Generate Obsidian vault (default) + optional HTML - -**Always generate the Obsidian vault and HTML** - they are the primary visualizations. Skip both if `--no-viz` (report + JSON only). - -```bash -python3 -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" -``` - -Also generate the HTML graph (always, unless `--no-viz`): - -```bash -python3 -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -if G.number_of_nodes() > 5000: - print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" -``` - -### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) - -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -python3 -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('.graphify_extract.json').read_text())) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -python3 -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - -### Step 7b - SVG export (only if --svg flag) - -```bash -python3 -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` - -### Step 7c - GraphML export (only if --graphml flag) - -```bash -python3 -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` - -### Step 7d - MCP server (only if --mcp flag) - -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` - -### Step 8 - Token reduction benchmark (only if total_words > 5000) - -If `total_words` from `.graphify_detect.json` is greater than 5,000, run: - -```bash -python3 -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. - ---- - -### Step 9 - Save manifest, update cost tracker, clean up, and report - -```bash -python3 -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('.graphify_detect.json').read_text()) -save_manifest(detect['files']) - -# Update cumulative cost tracker -extract = json.loads(Path('.graphify_extract.json').read_text()) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text()) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Tell the user: -``` -Graph complete. Outputs are in a hidden folder called graphify-out/ inside the directory you ran this on. - -The folder is hidden (dot prefix) so it won't show in Finder or a normal ls. -To see it: - Mac/Linux: ls -la graphify-out/ - VS Code: the Explorer panel shows hidden files by default - Finder: Cmd+Shift+. to toggle hidden files - -What's inside: - graphify-out/obsidian/ - open this folder as a vault in Obsidian (File > Open Vault) - graphify-out/GRAPH_REPORT.md - full audit report, also readable here in Claude - graphify-out/graph.json - persistent graph, query it later with /graphify query "..." - -Full path: PATH_TO_DIR/graphify-out/ -``` - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- - -## For --update (incremental re-extraction) - -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. - -```bash -python3 -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('.graphify_incremental.json').write_text(json.dumps(result)) -if new_total == 0: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -python3 -c " -import json -from pathlib import Path - -result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. - -Then: - -```bash -python3 -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction) - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -python3 -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None -new_extract = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` -Clean up after: `rm -f .graphify_old.json` - ---- - -## For --cluster-only - -Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -python3 -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). - ---- - -## For /graphify query - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -python3 -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -python3 -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - d = G.edges[u, v] - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` - -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -python3 -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='QUESTION', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='query', - source_nodes=SOURCE_NODES, # list of node labels cited, or [] -) -print('Query result saved to graphify-out/memory/') -" -``` - -Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - ---- - -## For /graphify path - -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -python3 -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -python3 -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - edge = G.edges[nid, path[i+1]] - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -python3 -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='Path from NODE_A to NODE_B', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='path_query', - source_nodes=PATH_NODES, # list of node labels on the path -) -print('Path result saved to graphify-out/memory/') -" -``` - ---- - -## For /graphify explain - -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -python3 -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -python3 -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - edge = G.edges[nid, neighbor] - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -python3 -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='Explain NODE_NAME', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='explain', - source_nodes=['NODE_NAME'], -) -print('Explanation saved to graphify-out/memory/') -" -``` - ---- - -## For /graphify add - -Fetch a URL and add it to the corpus, then update the graph. - -```bash -python3 -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- - -## For --watch - -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. - ---- - -## For git commit hook - -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - -If a post-commit hook already exists, graphify appends to it rather than replacing it. - ---- - -## For native CLAUDE.md integration - -Run once per project to make graphify always-on in Claude Code sessions: - -```bash -graphify claude install -``` - -This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. - -```bash -graphify claude uninstall # remove the section -``` - ---- - -## Honesty Rules - -- Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tests/test_install.py b/tests/test_install.py index 5cee59041..ac8f24b08 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -83,3 +83,77 @@ def test_claude_install_registers_claude_md(tmp_path): def test_codex_install_does_not_write_claude_md(tmp_path): _install(tmp_path, "codex") assert not (tmp_path / ".claude" / "CLAUDE.md").exists() + + +# --- always-on AGENTS.md install/uninstall tests --- + +def _agents_install(tmp_path, platform): + from graphify.__main__ import _agents_install as _install_fn + _install_fn(tmp_path, platform) + + +def _agents_uninstall(tmp_path): + from graphify.__main__ import _agents_uninstall as _uninstall_fn + _uninstall_fn(tmp_path) + + +def test_codex_agents_install_writes_agents_md(tmp_path): + _agents_install(tmp_path, "codex") + agents_md = tmp_path / "AGENTS.md" + assert agents_md.exists() + assert "graphify" in agents_md.read_text() + assert "GRAPH_REPORT.md" in agents_md.read_text() + + +def test_opencode_agents_install_writes_agents_md(tmp_path): + _agents_install(tmp_path, "opencode") + assert (tmp_path / "AGENTS.md").exists() + + +def test_claw_agents_install_writes_agents_md(tmp_path): + _agents_install(tmp_path, "claw") + assert (tmp_path / "AGENTS.md").exists() + + +def test_agents_install_idempotent(tmp_path): + """Installing twice does not duplicate the section.""" + _agents_install(tmp_path, "codex") + _agents_install(tmp_path, "codex") + content = (tmp_path / "AGENTS.md").read_text() + assert content.count("## graphify") == 1 + + +def test_agents_install_appends_to_existing(tmp_path): + """Installs into an existing AGENTS.md without overwriting other content.""" + agents_md = tmp_path / "AGENTS.md" + agents_md.write_text("# Existing rules\n\nDo not break things.\n") + _agents_install(tmp_path, "codex") + content = agents_md.read_text() + assert "Do not break things." in content + assert "## graphify" in content + + +def test_agents_uninstall_removes_section(tmp_path): + _agents_install(tmp_path, "codex") + _agents_uninstall(tmp_path) + agents_md = tmp_path / "AGENTS.md" + # File deleted when it only contained graphify section + assert not agents_md.exists() + + +def test_agents_uninstall_preserves_other_content(tmp_path): + """Uninstall keeps pre-existing content.""" + agents_md = tmp_path / "AGENTS.md" + agents_md.write_text("# Existing rules\n\nDo not break things.\n") + _agents_install(tmp_path, "codex") + _agents_uninstall(tmp_path) + assert agents_md.exists() + content = agents_md.read_text() + assert "Do not break things." in content + assert "## graphify" not in content + + +def test_agents_uninstall_no_op_when_not_installed(tmp_path, capsys): + _agents_uninstall(tmp_path) + out = capsys.readouterr().out + assert "nothing to do" in out