diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34d5dd7d..d93649c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,44 @@ on: workflow_dispatch: jobs: + skillgen-check: + # Fast lint-style guard: the skill files under graphify/ are generated from + # the fragments in tools/skillgen/. This fails if someone hand-edited a + # generated file or forgot to re-run the generator and bless expected/, and it + # runs the build-time validators that guard per-host coverage, the file_type + # enum, the monolith round-trips, and the always-on round-trips. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + # The audit-coverage, monolith-roundtrip, and always-on-roundtrip + # validators read blobs from origin/v8. A shallow checkout omits that + # ref, so fetch the full history here and the validators run for real + # (rather than skipping). The other jobs stay shallow. + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + with: + python-version: "3.12" + + # --frozen keeps uv from re-resolving and rewriting uv.lock as a side + # effect of `uv run`; the lock is committed and must not churn in CI. + - name: Check generated skill artifacts are up to date + run: uv run --frozen python -m tools.skillgen --check + + - name: Audit per-host v8 coverage + run: uv run --frozen python -m tools.skillgen --audit-coverage + + - name: Check the file_type enum is a singleton + run: uv run --frozen python -m tools.skillgen --schema-singleton + + - name: Round-trip the monoliths against v8 + run: uv run --frozen python -m tools.skillgen --monolith-roundtrip + + - name: Round-trip the always-on blocks against v8 + run: uv run --frozen python -m tools.skillgen --always-on-roundtrip + test: runs-on: ubuntu-latest strategy: @@ -22,13 +60,15 @@ jobs: with: python-version: ${{ matrix.python-version }} + # --frozen installs straight from the committed uv.lock without re-resolving + # or rewriting it, so CI never churns the lock. - name: Install dependencies - run: uv sync --all-extras + run: uv sync --all-extras --frozen - name: Run tests - run: uv run pytest tests/ -q --tb=short + run: uv run --frozen pytest tests/ -q --tb=short - name: Verify install works end-to-end run: | - uv run graphify --help - uv run graphify install + uv run --frozen graphify --help + uv run --frozen graphify install diff --git a/.gitignore b/.gitignore index d54e4036..fe5ff262 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,15 @@ graphify-out/ .graphify_python .claude/ skills/ +# The packaged skill bundles under graphify/skills/ are generated, committed +# artifacts (rendered by tools/skillgen). Keep them tracked even though the +# broad skills/ rule above ignores install-target skill dirs elsewhere. +!graphify/skills/ +!graphify/skills/** +# The skillgen core fragments are the human-edited source of the lean SKILL.md. +# A global "core" ignore (for core dumps) would otherwise drop them. +!tools/skillgen/fragments/core/ +!tools/skillgen/fragments/core/** docs/superpowers/ .vscode/ .kilo diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..5e762eb4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +# Run with: uv run pre-commit install (pre-commit is already a dev dependency) +# One-off across the tree: uv run pre-commit run --all-files +# +# The skillgen hook is the local anti-drift guard. The skill files under +# graphify/ are generated from the fragments in tools/skillgen/; a hand-edit to +# a generated file fails this check the same way CI does. Run +# `python -m tools.skillgen` then `--bless` to regenerate after a fragment edit. +repos: + - repo: local + hooks: + - id: skillgen-check + name: skillgen --check (generated skill artifacts are up to date) + entry: python -m tools.skillgen --check + language: system + pass_filenames: false + always_run: true + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.14 + hooks: + - id: ruff + args: ["--config", "pyproject.toml"] diff --git a/CHANGELOG.md b/CHANGELOG.md index ec335bcf..2ec8b877 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Full release notes with details on each version: [GitHub Releases](https://githu - Fix: `uv tool install graphifyy` / `pip install graphifyy` no longer fails to build on Linux/macOS — `tree-sitter-dm` (BYOND DreamMaker) ships only a Windows wheel, so on other platforms it compiled from source and aborted the entire install when a C toolchain or `python3-dev` headers were missing. It is now an optional extra (`graphifyy[dm]`, also in `[all]`) instead of a core dependency, so the default install needs no compiler (#1104). - **Upgrade note:** DreamMaker `.dm`/`.dme` users must reinstall with `graphifyy[dm]` (or `[all]`) to keep AST extraction — on `uv tool upgrade` the now-optional grammar is removed. `.dmi`/`.dmm`/`.dmf` parsing is unaffected (no tree-sitter dependency). - Fix: community IDs are now assigned by a total order (`(-size, sorted node IDs)`) so an identical grouping always gets identical IDs across runs — previously the equal-sized small communities that dominate a sparse graph were numbered by the partitioner's (not seed-stable) enumeration order, making a per-node community diff report large spurious "churn" even though the actual grouping was reproducible (#1090 follow-up) +- Fix: `graphify amp install` now writes the skill where Amp actually looks for it. It was landing in `.amp/skills/graphify` (project) and `~/.amp/skills/graphify` (user), neither of which Amp searches, so the skill never loaded. User-scope installs now go to `~/.config/agents/skills/graphify` and project installs to `.agents/skills/graphify`, and a stale `~/.amp/skills/graphify` from an older install is cleaned up on the next run. ## 0.8.27 (2026-05-31) diff --git a/graphify/__main__.py b/graphify/__main__.py index 8ac1f9f8..1f2affae 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -21,6 +21,20 @@ except Exception: _GRAPHIFY_OUT = os.environ.get("GRAPHIFY_OUT", "graphify-out") +def _always_on(basename: str) -> str: + """Read a packaged always-on instruction block from graphify/always_on/. + + The six always-on blocks (CLAUDE.md / AGENTS.md / GEMINI.md / VS Code + Copilot instructions / Antigravity rules / Kiro steering) live as committed + markdown next to this module, generated by tools/skillgen from a single + human-edited fragment and guarded against drift by ``skillgen --check``. The + installer injects them verbatim via ``_replace_or_append_section``, so the + bytes here must match the former triple-quoted constant exactly — the + always-on-roundtrip validator proves that. + """ + return (Path(__file__).parent / "always_on" / f"{basename}.md").read_text(encoding="utf-8") + + def _default_graph_path() -> str: return str(Path(_GRAPHIFY_OUT) / "graph.json") @@ -52,6 +66,15 @@ def _check_skill_version(skill_dst: Path) -> None: if not skill_dst.exists(): print(" warning: skill dir exists but SKILL.md is missing. Run 'graphify install' to repair.") return + # A progressive SKILL.md links to its references/ sidecar. If the body points + # at references/ but the dir is gone (manual delete, partial upgrade), the + # on-demand fragments won't load — flag it for repair. + try: + body = skill_dst.read_text(encoding="utf-8") + except OSError: + body = "" + if "references/" in body and not (skill_dst.parent / "references").exists(): + print(" warning: skill references/ sidecar is missing. Run 'graphify install' to repair.", file=sys.stderr) installed = version_file.read_text(encoding="utf-8").strip() if installed != __version__: print(f" warning: skill is from graphify {installed}, package is {__version__}. Run 'graphify install' to update.", file=sys.stderr) @@ -89,6 +112,11 @@ def _platform_skill_destination(platform_name: str, *, project: bool = False, pr return (project_dir or Path(".")) / ".devin" / "skills" / "graphify" / "SKILL.md" return Path.home() / ".config" / "devin" / "skills" / "graphify" / "SKILL.md" + if platform_name == "amp": + if project: + return (project_dir or Path(".")) / ".agents" / "skills" / "graphify" / "SKILL.md" + return Path.home() / ".config" / "agents" / "skills" / "graphify" / "SKILL.md" + if platform_name in ("antigravity", "antigravity-windows"): if project: return (project_dir or Path(".")) / ".agents" / "skills" / "graphify" / "SKILL.md" @@ -104,16 +132,99 @@ def _platform_skill_destination(platform_name: str, *, project: bool = False, pr return Path.home() / cfg["skill_dst"] +def _packaged_skill_refs_dir(platform_name: str) -> Path | None: + """Return the packaged references source dir for a progressive platform, else None. + + A platform opts into progressive disclosure by setting ``skill_refs`` in its + ``_PLATFORM_CONFIG`` entry. The value names a bundle under + ``graphify/skills//references/``. Reuse keys (e.g. trae-cn) point at + their twin's bundle. ``gemini`` has no config entry and is never progressive. + + Bundles ship one platform-group at a time. A host whose bundle directory + ``graphify/skills//`` is not in this build has not gone progressive + yet, so this returns None and the host installs today's monolithic SKILL.md + with no references/ sidecar. Only when the bundle directory IS present does + this return the references path; if that directory then lacks its + ``references/`` subdir, ``_copy_skill_file`` hard-fails (a malformed bundle, + the empty-sidecar regression the wheel-content test also guards). + """ + if platform_name == "gemini": + return None + bundle = _PLATFORM_CONFIG[platform_name].get("skill_refs") + if not bundle: + return None + bundle_dir = Path(__file__).parent / "skills" / bundle + if not bundle_dir.is_dir(): + return None + return bundle_dir / "references" + + +def _install_skill_references(skill_dst: Path, refs_src: Path) -> None: + """Atomically install a packaged references/ sidecar next to SKILL.md. + + Stages the packaged dir into ``references.tmp`` (copytree), drops any stale + ``references/`` already on disk, then ``os.replace``-renames the staged dir + into place. The rename is atomic on the same filesystem, so an interrupted + install never leaves a half-written references/ visible to the agent. + """ + refs_dst = skill_dst.parent / "references" + refs_staged = skill_dst.parent / "references.tmp" + if refs_staged.exists(): + shutil.rmtree(refs_staged) + try: + shutil.copytree(refs_src, refs_staged) + if refs_dst.exists(): + shutil.rmtree(refs_dst) + os.replace(refs_staged, refs_dst) + except Exception: + if refs_staged.exists(): + shutil.rmtree(refs_staged, ignore_errors=True) + raise + + def _copy_skill_file(platform_name: str, *, project: bool = False, project_dir: Path | None = None) -> Path: - """Copy a packaged skill file and write its version stamp.""" + """Copy a packaged skill file and write its version stamp. + + For progressive platforms (those with ``skill_refs`` set), the packaged + ``references/`` sidecar is installed alongside SKILL.md and the single + ``.graphify_version`` stamp covers both. For monolith platforms (no + ``skill_refs``), any orphan ``references/`` left by a prior progressive + install is removed so the on-disk layout matches the package. + """ skill_file = "skill.md" if platform_name == "gemini" else _PLATFORM_CONFIG[platform_name]["skill_file"] skill_src = Path(__file__).parent / skill_file if not skill_src.exists(): print(f"error: {skill_file} not found in package - reinstall graphify", file=sys.stderr) sys.exit(1) + refs_src = _packaged_skill_refs_dir(platform_name) + if refs_src is not None and not refs_src.exists(): + # Progressive platform declared a references bundle that is missing from + # the package. Fail loud rather than silently shipping an empty sidecar. + print( + f"error: references for '{platform_name}' not found in package " + f"({refs_src}) - reinstall graphify", + file=sys.stderr, + ) + sys.exit(1) + skill_dst = _platform_skill_destination(platform_name, project=project, project_dir=project_dir) skill_dst.parent.mkdir(parents=True, exist_ok=True) + + # Install the references/ sidecar (or clear an orphan one) BEFORE writing + # SKILL.md, so SKILL.md is the last artifact laid down. An install that is + # interrupted partway then leaves no SKILL.md rather than a SKILL.md that + # points at an absent references/ dir. + if refs_src is not None: + _install_skill_references(skill_dst, refs_src) + print(f" references -> {skill_dst.parent / 'references'}") + else: + # Monolith (or progressive-with-no-refs): clear any orphan references/. + orphan_refs = skill_dst.parent / "references" + if orphan_refs.exists(): + shutil.rmtree(orphan_refs) + + # SKILL.md last (crash-safety), via an atomic temp + rename. tmp_dst = skill_dst.with_suffix(skill_dst.suffix + ".tmp") try: shutil.copy(skill_src, tmp_dst) @@ -124,6 +235,7 @@ def _copy_skill_file(platform_name: str, *, project: bool = False, project_dir: except OSError: pass raise + (skill_dst.parent / ".graphify_version").write_text(__version__, encoding="utf-8") print(f" skill installed -> {skill_dst}") return skill_dst @@ -141,6 +253,10 @@ def _remove_skill_file(platform_name: str, *, project: bool = False, project_dir if version_file.exists(): version_file.unlink() removed = True + refs_dir = skill_dst.parent / "references" + if refs_dir.exists(): + shutil.rmtree(refs_dir) + removed = True for d in (skill_dst.parent, skill_dst.parent.parent, skill_dst.parent.parent.parent): try: d.rmdir() @@ -226,23 +342,28 @@ _PLATFORM_CONFIG: dict[str, dict] = { "skill_file": "skill.md", "skill_dst": Path(".claude") / "skills" / "graphify" / "SKILL.md", "claude_md": True, + "skill_refs": "claude", }, "codex": { "skill_file": "skill-codex.md", "skill_dst": Path(".agents") / "skills" / "graphify" / "SKILL.md", "claude_md": False, + "skill_refs": "codex", }, "opencode": { "skill_file": "skill-opencode.md", "skill_dst": Path(".config") / "opencode" / "skills" / "graphify" / "SKILL.md", "claude_md": False, + "skill_refs": "opencode", }, "kilo": { "skill_file": "skill-kilo.md", "skill_dst": Path(".config") / "kilo" / "skills" / "graphify" / "SKILL.md", "claude_md": False, + "skill_refs": "kilo", }, "aider": { + # Monolith: aider ships the full SKILL.md inline, no references/ sidecar. "skill_file": "skill-aider.md", "skill_dst": Path(".aider") / "graphify" / "SKILL.md", "claude_md": False, @@ -251,68 +372,89 @@ _PLATFORM_CONFIG: dict[str, dict] = { "skill_file": "skill-copilot.md", "skill_dst": Path(".copilot") / "skills" / "graphify" / "SKILL.md", "claude_md": False, + "skill_refs": "copilot", }, "claw": { "skill_file": "skill-claw.md", "skill_dst": Path(".openclaw") / "skills" / "graphify" / "SKILL.md", "claude_md": False, + "skill_refs": "claw", }, "droid": { "skill_file": "skill-droid.md", "skill_dst": Path(".factory") / "skills" / "graphify" / "SKILL.md", "claude_md": False, + "skill_refs": "droid", }, "trae": { "skill_file": "skill-trae.md", "skill_dst": Path(".trae") / "skills" / "graphify" / "SKILL.md", "claude_md": False, + "skill_refs": "trae", }, "trae-cn": { + # Reuses trae's split bundle (same skill body + references). "skill_file": "skill-trae.md", "skill_dst": Path(".trae-cn") / "skills" / "graphify" / "SKILL.md", "claude_md": False, + "skill_refs": "trae", }, "hermes": { + # Reuses claw's split bundle. "skill_file": "skill-claw.md", "skill_dst": Path(".hermes") / "skills" / "graphify" / "SKILL.md", "claude_md": False, + "skill_refs": "claw", }, "kiro": { "skill_file": "skill-kiro.md", "skill_dst": Path(".kiro") / "skills" / "graphify" / "SKILL.md", "claude_md": False, + "skill_refs": "kiro", }, "pi": { "skill_file": "skill-pi.md", "skill_dst": Path(".pi") / "agent" / "skills" / "graphify" / "SKILL.md", "claude_md": False, + "skill_refs": "pi", }, "antigravity": { + # Rides claude's split bundle (shares skill.md). "skill_file": "skill.md", "skill_dst": Path(".agents") / "skills" / "graphify" / "SKILL.md", "claude_md": False, + "skill_refs": "claude", }, "antigravity-windows": { + # Rides windows' split bundle. "skill_file": "skill-windows.md", "skill_dst": Path(".agents") / "skills" / "graphify" / "SKILL.md", "claude_md": False, + "skill_refs": "windows", }, "windows": { "skill_file": "skill-windows.md", "skill_dst": Path(".claude") / "skills" / "graphify" / "SKILL.md", "claude_md": True, + "skill_refs": "windows", }, "kimi": { + # Reuses claude's split bundle (shares skill.md). "skill_file": "skill.md", "skill_dst": Path(".kimi") / "skills" / "graphify" / "SKILL.md", "claude_md": False, + "skill_refs": "claude", }, "amp": { + # Amp searches .agents/skills (project) and ~/.config/agents/skills (user), + # not .amp/skills. The user-scope path is set in _platform_skill_destination. "skill_file": "skill-amp.md", - "skill_dst": Path(".amp") / "skills" / "graphify" / "SKILL.md", + "skill_dst": Path(".agents") / "skills" / "graphify" / "SKILL.md", "claude_md": False, + "skill_refs": "amp", }, "devin": { + # Monolith: devin ships the full SKILL.md inline, no references/ sidecar. "skill_file": "skill-devin.md", # User scope: ~/.config/devin/skills/graphify/SKILL.md # Project scope: .devin/skills/graphify/SKILL.md (overridden in _platform_skill_destination) @@ -441,50 +583,21 @@ def _print_install_usage() -> None: print(f"Platforms: {platforms}") -_CLAUDE_MD_SECTION = """\ -## graphify - -This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. - -Rules: -- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. -- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. -- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. -- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). -""" +# The always-on instruction blocks are packaged markdown under graphify/always_on/, +# generated by tools/skillgen and guarded by `skillgen --check`. Reading them at +# load keeps the install-string / issue-#580 contract byte-for-byte while letting +# a human edit one fragment instead of a triple-quoted literal here. +_CLAUDE_MD_SECTION = _always_on("claude-md") _CLAUDE_MD_MARKER = "## graphify" # AGENTS.md section for Codex, OpenCode, and OpenClaw. # All three platforms read AGENTS.md in the project root for persistent instructions. -_AGENTS_MD_SECTION = """\ -## graphify - -This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. - -When the user types `/graphify`, invoke the `skill` tool with `skill: "graphify"` before doing anything else. - -Rules: -- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. -- Dirty graphify-out/ files are expected after hooks or incremental updates; dirty graph files are not a reason to skip graphify. Only skip graphify if the task is about stale or incorrect graph output, or the user explicitly says not to use it. -- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. -- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. -- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). -""" +_AGENTS_MD_SECTION = _always_on("agents-md") _AGENTS_MD_MARKER = "## graphify" -_GEMINI_MD_SECTION = """\ -## graphify - -This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. - -Rules: -- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. -- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. -- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. -- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). -""" +_GEMINI_MD_SECTION = _always_on("gemini-md") _GEMINI_MD_MARKER = "## graphify" @@ -601,35 +714,37 @@ def gemini_uninstall(project_dir: Path | None = None, *, project: bool = False) _VSCODE_INSTRUCTIONS_MARKER = "## graphify" -_VSCODE_INSTRUCTIONS_SECTION = """\ -## graphify - -For any question about this repo's architecture, structure, components, or how to add/modify/find -code, your first action should be `graphify query ""` when `graphify-out/graph.json` -exists. Use `graphify path "" ""` for relationship questions and `graphify explain ""` -for focused-concept questions. These return a scoped subgraph, usually much smaller than the full -report or raw grep output. - -Triggers: "how do I…", "where is…", "what does … do", "add/modify a ", -"explain the architecture", or anything that depends on how files or classes relate. - -If `graphify-out/wiki/index.md` exists, use it for broad navigation. Read `graphify-out/GRAPH_REPORT.md` -only for broad architecture review or when query/path/explain do not surface enough context. Only read -source files when (a) modifying/debugging specific code, (b) the graph lacks the needed detail, or -(c) the graph is missing or stale. - -Type `/graphify` in Copilot Chat to build or update the graph. -""" +_VSCODE_INSTRUCTIONS_SECTION = _always_on("vscode-instructions") def vscode_install(project_dir: Path | None = None) -> None: """Install graphify skill for VS Code Copilot Chat + write .github/copilot-instructions.md.""" skill_src = Path(__file__).parent / "skill-vscode.md" + refs_bundle = "vscode" if not skill_src.exists(): skill_src = Path(__file__).parent / "skill-copilot.md" + refs_bundle = "copilot" skill_dst = Path.home() / ".copilot" / "skills" / "graphify" / "SKILL.md" skill_dst.parent.mkdir(parents=True, exist_ok=True) - shutil.copy(skill_src, skill_dst) + tmp_dst = skill_dst.with_suffix(skill_dst.suffix + ".tmp") + try: + shutil.copy(skill_src, tmp_dst) + os.replace(tmp_dst, skill_dst) + except Exception: + try: + tmp_dst.unlink(missing_ok=True) + except OSError: + pass + raise + # Progressive-capable: install the packaged references/ sidecar when present. + refs_src = Path(__file__).parent / "skills" / refs_bundle / "references" + if refs_src.exists(): + _install_skill_references(skill_dst, refs_src) + print(f" references -> {skill_dst.parent / 'references'}") + else: + orphan_refs = skill_dst.parent / "references" + if orphan_refs.exists(): + shutil.rmtree(orphan_refs) (skill_dst.parent / ".graphify_version").write_text(__version__, encoding="utf-8") print(f" skill installed -> {skill_dst}") @@ -665,6 +780,9 @@ def vscode_uninstall(project_dir: Path | None = None) -> None: version_file = skill_dst.parent / ".graphify_version" if version_file.exists(): version_file.unlink() + refs_dir = skill_dst.parent / "references" + if refs_dir.exists(): + shutil.rmtree(refs_dir) for d in ( skill_dst.parent, skill_dst.parent.parent, @@ -695,22 +813,7 @@ def vscode_uninstall(project_dir: Path | None = None) -> None: _ANTIGRAVITY_RULES_PATH = Path(".agents") / "rules" / "graphify.md" _ANTIGRAVITY_WORKFLOW_PATH = Path(".agents") / "workflows" / "graphify.md" -_ANTIGRAVITY_RULES = """\ ---- -trigger: always_on -description: Consult the graphify knowledge graph at graphify-out/ for codebase and architecture questions. ---- - -## graphify - -This project has a graphify knowledge graph at graphify-out/. - -Rules: -- For codebase or architecture questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (CLI) or `query_graph` (MCP). Use `graphify path "" ""` / `shortest_path` for relationships and `graphify explain ""` / `get_node` for focused concepts. These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. -- If graphify-out/wiki/index.md exists, navigate it instead of reading raw files -- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context -- After modifying code files in this session, run `graphify update .` to keep the graph current (AST-only, no API cost) -""" +_ANTIGRAVITY_RULES = _always_on("antigravity-rules") _ANTIGRAVITY_WORKFLOW = """\ --- @@ -726,17 +829,7 @@ If no path argument is given, use `.` (current directory). """ -_KIRO_STEERING = """\ ---- -inclusion: always ---- - -graphify: A knowledge graph of this project lives in `graphify-out/`. \ -For codebase, architecture, or dependency questions, when `graphify-out/graph.json` exists, \ -first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). \ -These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. \ -Read `GRAPH_REPORT.md` only for broad architecture review or when those commands do not surface enough context. -""" +_KIRO_STEERING = _always_on("kiro-steering") _KIRO_STEERING_MARKER = "graphify: A knowledge graph of this project" @@ -873,6 +966,9 @@ def _antigravity_uninstall(project_dir: Path, *, project: bool = False) -> None: version_file = skill_dst.parent / ".graphify_version" if version_file.exists(): version_file.unlink() + refs_dir = skill_dst.parent / "references" + if refs_dir.exists(): + shutil.rmtree(refs_dir) for d in ( skill_dst.parent, skill_dst.parent.parent, @@ -1341,6 +1437,35 @@ def _agents_install(project_dir: Path, platform: str) -> None: ) +def _amp_legacy_cleanup() -> None: + """Best-effort removal of the pre-fix ~/.amp/skills/graphify install dir. + + Older graphify versions wrote the Amp skill to ~/.amp/skills, which Amp does + not search. Clean it up on install so a stale, never-loaded copy does not + linger. Failures are ignored (the new path is what matters). + """ + legacy = Path.home() / ".amp" / "skills" / "graphify" + if legacy.exists(): + shutil.rmtree(legacy, ignore_errors=True) + if not legacy.exists(): + print(f" legacy removed -> {legacy}") + + +def _amp_install(project_dir: Path | None = None) -> None: + """User-scope Amp install: skill into ~/.config/agents/skills + AGENTS.md.""" + _amp_legacy_cleanup() + _copy_skill_file("amp") + _agents_install(project_dir or Path("."), "amp") + + +def _amp_uninstall(project_dir: Path | None = None) -> None: + """User-scope Amp uninstall: remove the skill and the AGENTS.md section.""" + removed = _remove_skill_file("amp") + if removed: + print("skill removed") + _agents_uninstall(project_dir or Path("."), platform="amp") + + def _project_install(platform_name: str, project_dir: Path | None = None) -> None: """Install platform skill/config files in the current project.""" project_dir = project_dir or Path(".") @@ -1585,6 +1710,9 @@ def uninstall_all(project_dir: Path | None = None, purge: bool = False) -> None: _antigravity_uninstall(pd) # AGENTS.md covers: codex, aider, opencode, claw, droid, trae, trae-cn, hermes, copilot _agents_uninstall(pd) + # Amp also drops a user-scope skill at ~/.config/agents/skills, which the + # AGENTS.md cleanup above does not touch. + _remove_skill_file("amp") _uninstall_opencode_plugin(pd) _uninstall_codex_hook(pd) @@ -1711,7 +1839,10 @@ def main() -> None: # Deduplicate paths so platforms sharing the same install dir don't warn twice. _silent_cmds = {"install", "uninstall", "hook-check"} if not any(arg in _silent_cmds for arg in sys.argv): - for skill_dst in {Path.home() / cfg["skill_dst"] for cfg in _PLATFORM_CONFIG.values()}: + # Resolve each platform's real user-scope destination so per-platform + # overrides (gemini, opencode, devin, antigravity, amp) check the dir + # they actually install into, not the bare cfg['skill_dst']. + for skill_dst in {_platform_skill_destination(name) for name in _PLATFORM_CONFIG}: _check_skill_version(skill_dst) if len(sys.argv) >= 2 and sys.argv[1] in ("-v", "--version", "version"): @@ -1722,7 +1853,7 @@ def main() -> None: print("Usage: graphify ") print() print("Commands:") - print(" install [--platform P] copy skill to platform config dir (claude|windows|codex|opencode|aider|claw|droid|trae|trae-cn|gemini|cursor|antigravity|hermes|kiro|pi|devin)") + print(" install [--platform P] copy skill to platform config dir (claude|windows|codex|opencode|aider|amp|claw|droid|trae|trae-cn|gemini|cursor|antigravity|hermes|kiro|pi|devin)") print(" uninstall remove graphify from all detected platforms in one shot") print(" --purge also delete graphify-out/ directory") print(" path \"A\" \"B\" shortest path between two nodes in graph.json") @@ -2075,7 +2206,22 @@ def main() -> None: else: print("Usage: graphify pi [install|uninstall]", file=sys.stderr) sys.exit(1) - elif cmd in ("aider", "amp", "codex", "opencode", "claw", "droid", "trae", "trae-cn", "hermes"): + elif cmd == "amp": + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd == "install": + if "--project" in sys.argv[3:]: + _project_install("amp", Path(".")) + else: + _amp_install(Path(".")) + elif subcmd == "uninstall": + if "--project" in sys.argv[3:]: + _project_uninstall("amp", Path(".")) + else: + _amp_uninstall(Path(".")) + else: + print("Usage: graphify amp [install|uninstall]", file=sys.stderr) + sys.exit(1) + elif cmd in ("aider", "codex", "opencode", "claw", "droid", "trae", "trae-cn", "hermes"): subcmd = sys.argv[2] if len(sys.argv) > 2 else "" if subcmd == "install": if "--project" in sys.argv[3:]: diff --git a/graphify/always_on/agents-md.md b/graphify/always_on/agents-md.md new file mode 100644 index 00000000..20cff728 --- /dev/null +++ b/graphify/always_on/agents-md.md @@ -0,0 +1,12 @@ +## graphify + +This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. + +When the user types `/graphify`, invoke the `skill` tool with `skill: "graphify"` before doing anything else. + +Rules: +- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- Dirty graphify-out/ files are expected after hooks or incremental updates; dirty graph files are not a reason to skip graphify. Only skip graphify if the task is about stale or incorrect graph output, or the user explicitly says not to use it. +- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. +- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/graphify/always_on/antigravity-rules.md b/graphify/always_on/antigravity-rules.md new file mode 100644 index 00000000..0fc78641 --- /dev/null +++ b/graphify/always_on/antigravity-rules.md @@ -0,0 +1,14 @@ +--- +trigger: always_on +description: Consult the graphify knowledge graph at graphify-out/ for codebase and architecture questions. +--- + +## graphify + +This project has a graphify knowledge graph at graphify-out/. + +Rules: +- For codebase or architecture questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (CLI) or `query_graph` (MCP). Use `graphify path "" ""` / `shortest_path` for relationships and `graphify explain ""` / `get_node` for focused concepts. These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. +- If graphify-out/wiki/index.md exists, navigate it instead of reading raw files +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context +- After modifying code files in this session, run `graphify update .` to keep the graph current (AST-only, no API cost) diff --git a/graphify/always_on/claude-md.md b/graphify/always_on/claude-md.md new file mode 100644 index 00000000..417efeb2 --- /dev/null +++ b/graphify/always_on/claude-md.md @@ -0,0 +1,9 @@ +## graphify + +This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. + +Rules: +- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. +- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/graphify/always_on/gemini-md.md b/graphify/always_on/gemini-md.md new file mode 100644 index 00000000..417efeb2 --- /dev/null +++ b/graphify/always_on/gemini-md.md @@ -0,0 +1,9 @@ +## graphify + +This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. + +Rules: +- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. +- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/graphify/always_on/kiro-steering.md b/graphify/always_on/kiro-steering.md new file mode 100644 index 00000000..cb6f4543 --- /dev/null +++ b/graphify/always_on/kiro-steering.md @@ -0,0 +1,5 @@ +--- +inclusion: always +--- + +graphify: A knowledge graph of this project lives in `graphify-out/`. For codebase, architecture, or dependency questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. Read `GRAPH_REPORT.md` only for broad architecture review or when those commands do not surface enough context. diff --git a/graphify/always_on/vscode-instructions.md b/graphify/always_on/vscode-instructions.md new file mode 100644 index 00000000..9cb983c9 --- /dev/null +++ b/graphify/always_on/vscode-instructions.md @@ -0,0 +1,17 @@ +## graphify + +For any question about this repo's architecture, structure, components, or how to add/modify/find +code, your first action should be `graphify query ""` when `graphify-out/graph.json` +exists. Use `graphify path "" ""` for relationship questions and `graphify explain ""` +for focused-concept questions. These return a scoped subgraph, usually much smaller than the full +report or raw grep output. + +Triggers: "how do I…", "where is…", "what does … do", "add/modify a ", +"explain the architecture", or anything that depends on how files or classes relate. + +If `graphify-out/wiki/index.md` exists, use it for broad navigation. Read `graphify-out/GRAPH_REPORT.md` +only for broad architecture review or when query/path/explain do not surface enough context. Only read +source files when (a) modifying/debugging specific code, (b) the graph lacks the needed detail, or +(c) the graph is missing or stale. + +Type `/graphify` in Copilot Chat to build or update the graph. diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index 2db7ea36..aad4ff95 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -1,6 +1,6 @@ --- name: graphify -description: "any input (code, docs, papers, images) → knowledge graph → clustered communities → HTML + JSON + audit report. Use when user asks any question about a codebase, project content, architecture, or file relationships — especially if graphify-out/ exists. Provides persistent graph with god nodes, community detection, and BFS/DFS query tools." +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." trigger: /graphify --- @@ -254,7 +254,7 @@ Process each file one at a time. For each file: - INFERRED: reasonable inference (shared structure, implied dependency) - AMBIGUOUS: uncertain — flag it, do not omit - Code files: semantic edges AST cannot find. Do not re-extract imports. - - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms). Do NOT invent file_types like `concept`. When adding `calls` edges: source is caller, target is callee. + - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. When adding `calls` edges: source is caller, target is callee. - Image files: use vision — understand what the image IS, not just OCR - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges - Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. @@ -263,7 +263,7 @@ Process each file one at a time. For each file: 3. Accumulate results across all files Schema for each file's output: -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} After processing all files, write the accumulated result to `.graphify_semantic_new.json`. diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index 027095eb..9cad76fc 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -1,6 +1,6 @@ --- name: graphify -description: "any input (code, docs, papers, images) → knowledge graph → clustered communities → HTML + JSON + audit report. Use when user asks any question about a codebase, project content, architecture, or file relationships — especially if graphify-out/ exists. Provides persistent graph with god nodes, community detection, and BFS/DFS query tools." +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." trigger: /graphify --- @@ -13,8 +13,13 @@ Turn any folder of files into a navigable knowledge graph with community detecti ``` /graphify # full pipeline on current directory → Obsidian vault /graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON /graphify --html # (HTML is generated by default - this flag is a no-op) @@ -24,6 +29,8 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j /graphify --mcp # start MCP stdio server for agent access /graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) /graphify add # fetch URL, save to ./raw, update graph /graphify add --author "Name" # tag who wrote it /graphify add --contributor "Name" # tag who added it to the corpus @@ -36,29 +43,24 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes → one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. ## What You Must Do When Invoked If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + If no path was given, use `.` (current directory). Do not ask the user for a path. -If `graphify-out/` exists, use `graphify query`, `graphify explain`, or `graphify path` for orientation before broad grep, rg, or multi-file reads. Dirty `graphify-out/` artifacts are expected after hooks or incremental updates; dirty graph files are not a reason to skip Graphify. Only skip Graphify if the task is specifically about stale or incorrect graph output, or the user explicitly says not to use it. +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. Follow these steps in order. Do not skip steps. +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + ### Step 1 - Ensure graphify is installed ```bash @@ -93,6 +95,8 @@ fi # Write interpreter path for all subsequent steps (persists across invocations) mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -107,8 +111,8 @@ import json from graphify.detect import detect from pathlib import Path result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) -" > .graphify_detect.json +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -127,58 +131,31 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. - Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. -### Step 2.5 - Transcribe video / audio files (only if video files detected) +### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. ### Step 3 - Extract entities and relationships **Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** @@ -196,16 +173,16 @@ from pathlib import Path import json code_files = [] -detect = json.loads(Path('.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files) - Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: - Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') " ``` @@ -214,10 +191,10 @@ else: **Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. -**MANDATORY: You MUST use the Task tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Task tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `.graphify_detect.json` +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` - Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) - Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) - Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" @@ -232,104 +209,58 @@ import json from graphify.cache import check_semantic_cache from pathlib import Path -detect = json.loads(Path('.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) all_files = [f for files in detect['files'].values() for f in files] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) if cached_nodes or cached_edges or cached_hyperedges: - Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " ``` -Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. -**Step B2 - Dispatch ALL subagents in a single message (Amp)** +**Step B2 - Dispatch ALL subagents in a single message** -> **Amp platform:** Uses the `Task` tool to dispatch parallel subagents. Multiple `Task` calls issued in the **same response** run concurrently — that is how parallelism is achieved on Amp. Do not issue `Task` calls one-at-a-time across multiple turns. +> Uses the `Task` tool for parallel subagent dispatch. +> Call `Task` once per chunk — ALL in the same response so they run in parallel. -Call `Task` once per chunk — ALL in the same response so they run in parallel. Build each call by wrapping the extraction prompt below in task-delegation framing: +Pass the extraction prompt as the task description: ``` -Task(description="graphify extraction chunk CHUNK_NUM/TOTAL_CHUNKS", prompt="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt below, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") +Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") ``` -Each `Task` subagent returns its result as a single message when it finishes. Collect every subagent's returned message, parse it as JSON, and accumulate nodes/edges/hyperedges across all results into `.graphify_semantic_new.json`. - -If a subagent returns output that is not valid JSON (e.g. wraps it in prose or markdown fences), strip the wrapper and parse the inner JSON. If parsing still fails, log the chunk to `.graphify_failed_chunks.txt` and continue with the remaining chunks — do not abort the run. - -The extraction prompt each subagent receives (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE): +Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" ``` -You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. -Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. -Files (chunk CHUNK_NUM of TOTAL_CHUNKS): -FILE_LIST +Subagent prompt template: -Rules: -- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") -- INFERRED: reasonable inference (shared data structure, implied dependency) -- AMBIGUOUS: uncertain - flag for review, do not omit - -Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). - Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). Do NOT invent file_types like `concept` — valid values are only `code|document|paper|image|rationale`. -Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. -Image files: use vision to understand what the image IS - do not just OCR. - UI screenshot: layout patterns, design decisions, key elements, purpose. - Chart: metric, trend/insight, data source. - Tweet/post: claim as node, author, concepts mentioned. - Diagram: components and connections. - Research figure: what it demonstrates, method, result. - Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. - -DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, - shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - -Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: -- Two functions that both validate user input but never call each other -- A class in code and a concept in a paper that describe the same algorithm -- Two error types that handle the same failure mode differently -Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. - -Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: -- All classes that implement a common protocol or interface -- All functions in an authentication flow (even if they don't all call each other) -- All concepts from a paper section that form one coherent idea -Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. - -If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, - contributor onto every node from that file. - -confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: -- EXTRACTED edges: confidence_score = 1.0 always -- INFERRED edges: reason about each edge individually. - Direct structural evidence (shared data structure, clear dependency): 0.8-0.9. - Reasonable inference with some uncertainty: 0.6-0.7. - Weak or speculative: 0.4-0.5. Most edges should be 0.6-0.9, not 0.5. -- AMBIGUOUS edges: 0.1-0.3 - -Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} -``` +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. **Step B3 - Collect, cache, and merge** Wait for all subagents. For each result: - Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal - If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. - If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort -If more than half the chunks failed or are missing, stop and tell the user to re-run. +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. -Merge all chunk files into `.graphify_semantic_new.json`. Amp's `Task` tool does not expose per-call token usage in the result, so leave the placeholder zero values in each chunk's `input_tokens`/`output_tokens` fields and report aggregate cost as "not available" in the run summary. Then run: +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob @@ -339,7 +270,7 @@ chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: - d = json.loads(Path(c).read_text()) + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) @@ -348,7 +279,7 @@ for c in chunks: Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') " ``` @@ -360,20 +291,20 @@ import json from graphify.cache import save_semantic_cache from pathlib import Path -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) print(f'Cached {saved} files') " ``` -Merge cached + new results into `.graphify_semantic.json`: +Merge cached + new results into `graphify-out/.graphify_semantic.json`: ```bash $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path -cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) @@ -392,11 +323,11 @@ merged = { 'input_tokens': new.get('input_tokens', 0), 'output_tokens': new.get('output_tokens', 0), } -Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction @@ -405,8 +336,8 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from pathlib import Path -ast = json.loads(Path('.graphify_ast.json').read_text()) -sem = json.loads(Path('.graphify_semantic.json').read_text()) +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} @@ -425,7 +356,7 @@ merged = { 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } -Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") total = len(merged_nodes) edges = len(merged_edges) print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') @@ -434,6 +365,8 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + ```bash mkdir -p graphify-out $(cat graphify-out/.graphify_python) -c " @@ -445,8 +378,8 @@ from graphify.report import generate from graphify.export import to_json from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = cluster(G) @@ -458,8 +391,8 @@ labels = {cid: 'Community ' + str(cid) for cid in communities} # Placeholder questions - regenerated with real labels in Step 5 questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") to_json(G, communities, 'graphify-out/graph.json') analysis = { @@ -469,7 +402,7 @@ analysis = { 'surprises': surprises, 'questions': questions, } -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") if G.number_of_nodes() == 0: print('ERROR: Graph is empty - extraction produced no nodes.') print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') @@ -484,7 +417,7 @@ Replace INPUT_PATH with the actual path. ### Step 5 - Label communities -Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). Then regenerate the report and save the labels for the visualizer: @@ -497,9 +430,9 @@ from graphify.analyze import god_nodes, surprising_connections, suggest_question from graphify.report import generate from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} @@ -512,9 +445,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') " ``` @@ -528,178 +461,23 @@ Replace INPUT_PATH with the actual path. If `--obsidian` was given: +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -if G.number_of_nodes() > 5000: - print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` -### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('.graphify_extract.json').read_text())) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - -### Step 7b - SVG export (only if --svg flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` - -### Step 7c - GraphML export (only if --graphml flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` - -### Step 7d - MCP server (only if --mcp flag) - -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` - -### Step 8 - Token reduction benchmark (only if total_words > 5000) - -If `total_words` from `.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. --- @@ -713,17 +491,19 @@ from datetime import datetime, timezone from graphify.detect import save_manifest # Save manifest for --update -detect = json.loads(Path('.graphify_detect.json').read_text()) -save_manifest(detect['files']) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) # Update cumulative cost tracker -extract = json.loads(Path('.graphify_extract.json').read_text()) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) cost_path = Path('graphify-out/cost.json') if cost_path.exists(): - cost = json.loads(cost_path.read_text()) + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) else: cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} @@ -735,12 +515,12 @@ cost['runs'].append({ }) cost['total_input_tokens'] += input_tok cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_chunk_*.json +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json rm -f graphify-out/.needs_update 2>/dev/null || true ``` @@ -775,521 +555,51 @@ The graph is the map. Your job after the pipeline is to be the guide. --- -## For --update (incremental re-extraction) +## Interpreter guard for subcommands -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" fi ``` +## For --update and --cluster-only -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction) - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None -new_extract = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` -Clean up after: `rm -f .graphify_old.json` - ---- - -## For --cluster-only - -Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. --- ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" +graphify query "" ``` -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - -Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. --- -## For /graphify path +## For /graphify add and --watch -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. --- -## For /graphify explain +## For the commit hook and native AGENTS.md integration -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- - -## For /graphify add - -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text - ---- - -## For --watch - -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. - ---- - -## For git commit hook - -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - -If a post-commit hook already exists, graphify appends to it rather than replacing it. - ---- - -## For native AGENTS.md integration - -Run once per project to make graphify always-on in Amp sessions: - -```bash -graphify amp install -``` - -This writes a `## graphify` section to the local `AGENTS.md` that instructs Amp to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. - -```bash -graphify amp uninstall # remove the section -``` +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's AGENTS.md, see `references/hooks.md`. --- diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 10462cb7..bad0a0d5 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -1,6 +1,6 @@ --- name: graphify -description: "any input (code, docs, papers, images) → knowledge graph → clustered communities → HTML + JSON + audit report. Use when user asks any question about a codebase, project content, architecture, or file relationships — especially if graphify-out/ exists. Provides persistent graph with god nodes, community detection, and BFS/DFS query tools." +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." trigger: /graphify --- @@ -13,8 +13,13 @@ Turn any folder of files into a navigable knowledge graph with community detecti ``` /graphify # full pipeline on current directory → Obsidian vault /graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON /graphify --html # (HTML is generated by default - this flag is a no-op) @@ -24,6 +29,8 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j /graphify --mcp # start MCP stdio server for agent access /graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) /graphify add # fetch URL, save to ./raw, update graph /graphify add --author "Name" # tag who wrote it /graphify add --contributor "Name" # tag who added it to the corpus @@ -36,27 +43,24 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes → one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. ## What You Must Do When Invoked If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + If no path was given, use `.` (current directory). Do not ask the user for a path. +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + Follow these steps in order. Do not skip steps. +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + ### Step 1 - Ensure graphify is installed ```bash @@ -91,6 +95,8 @@ fi # Write interpreter path for all subsequent steps (persists across invocations) mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -105,8 +111,8 @@ import json from graphify.detect import detect from pathlib import Path result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) -" > .graphify_detect.json +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -125,58 +131,31 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. - Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. -### Step 2.5 - Transcribe video / audio files (only if video files detected) +### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. ### Step 3 - Extract entities and relationships **Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** @@ -194,16 +173,16 @@ from pathlib import Path import json code_files = [] -detect = json.loads(Path('.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files) - Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: - Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') " ``` @@ -212,9 +191,13 @@ else: **Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. -> **OpenClaw platform:** Multi-agent support is still early on OpenClaw. Extraction runs sequentially — you read and extract each file yourself. This is slower than parallel platforms but fully reliable. +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** -Print: `"Semantic extraction: N files (sequential — OpenClaw)"` +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" **Step B0 - Check extraction cache first** @@ -226,52 +209,59 @@ import json from graphify.cache import check_semantic_cache from pathlib import Path -detect = json.loads(Path('.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) all_files = [f for files in detect['files'].values() for f in files] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) if cached_nodes or cached_edges or cached_hyperedges: - Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " ``` -Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. -**Step B2 - Sequential extraction (OpenClaw)** +**Step B2 - Dispatch ALL subagents in a single message** -Process each file one at a time. For each file: +Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. -1. Read the file contents -2. Extract nodes, edges, and hyperedges applying the same rules: - - EXTRACTED: relationship explicit in source (import, call, citation) - - INFERRED: reasonable inference (shared structure, implied dependency) - - AMBIGUOUS: uncertain — flag it, do not omit - - Code files: semantic edges AST cannot find. Do not re-extract imports. - - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms). Do NOT invent file_types like `concept`. When adding `calls` edges: source is caller, target is callee. - - Image files: use vision — understand what the image IS, not just OCR - - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges - - Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. - - Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. - - confidence_score REQUIRED on every edge: EXTRACTED=1.0, INFERRED=0.6-0.9 (reason individually), AMBIGUOUS=0.1-0.3 -3. Accumulate results across all files +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. -Schema for each file's output: -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +Concrete example for 3 chunks: +``` +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] +``` +All three in one message. Not three separate messages. -After processing all files, write the accumulated result to `.graphify_semantic_new.json`. +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -**Step B3 - Cache and merge** +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` -For the accumulated result: +Subagent prompt template: -If more than half the chunks failed, stop and tell the user. +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash @@ -283,7 +273,7 @@ chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: - d = json.loads(Path(c).read_text()) + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) @@ -292,7 +282,7 @@ for c in chunks: Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') " ``` @@ -304,20 +294,20 @@ import json from graphify.cache import save_semantic_cache from pathlib import Path -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) print(f'Cached {saved} files') " ``` -Merge cached + new results into `.graphify_semantic.json`: +Merge cached + new results into `graphify-out/.graphify_semantic.json`: ```bash $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path -cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) @@ -336,11 +326,11 @@ merged = { 'input_tokens': new.get('input_tokens', 0), 'output_tokens': new.get('output_tokens', 0), } -Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction @@ -349,8 +339,8 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from pathlib import Path -ast = json.loads(Path('.graphify_ast.json').read_text()) -sem = json.loads(Path('.graphify_semantic.json').read_text()) +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} @@ -369,7 +359,7 @@ merged = { 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } -Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") total = len(merged_nodes) edges = len(merged_edges) print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') @@ -378,6 +368,8 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + ```bash mkdir -p graphify-out $(cat graphify-out/.graphify_python) -c " @@ -389,8 +381,8 @@ from graphify.report import generate from graphify.export import to_json from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = cluster(G) @@ -402,8 +394,8 @@ labels = {cid: 'Community ' + str(cid) for cid in communities} # Placeholder questions - regenerated with real labels in Step 5 questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") to_json(G, communities, 'graphify-out/graph.json') analysis = { @@ -413,7 +405,7 @@ analysis = { 'surprises': surprises, 'questions': questions, } -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") if G.number_of_nodes() == 0: print('ERROR: Graph is empty - extraction produced no nodes.') print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') @@ -428,7 +420,7 @@ Replace INPUT_PATH with the actual path. ### Step 5 - Label communities -Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). Then regenerate the report and save the labels for the visualizer: @@ -441,9 +433,9 @@ from graphify.analyze import god_nodes, surprising_connections, suggest_question from graphify.report import generate from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} @@ -456,9 +448,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') " ``` @@ -472,178 +464,23 @@ Replace INPUT_PATH with the actual path. If `--obsidian` was given: +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -if G.number_of_nodes() > 5000: - print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` -### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('.graphify_extract.json').read_text())) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - -### Step 7b - SVG export (only if --svg flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` - -### Step 7c - GraphML export (only if --graphml flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` - -### Step 7d - MCP server (only if --mcp flag) - -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` - -### Step 8 - Token reduction benchmark (only if total_words > 5000) - -If `total_words` from `.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. --- @@ -657,17 +494,19 @@ from datetime import datetime, timezone from graphify.detect import save_manifest # Save manifest for --update -detect = json.loads(Path('.graphify_detect.json').read_text()) -save_manifest(detect['files']) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) # Update cumulative cost tracker -extract = json.loads(Path('.graphify_extract.json').read_text()) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) cost_path = Path('graphify-out/cost.json') if cost_path.exists(): - cost = json.loads(cost_path.read_text()) + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) else: cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} @@ -679,12 +518,12 @@ cost['runs'].append({ }) cost['total_input_tokens'] += input_tok cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_chunk_*.json +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json rm -f graphify-out/.needs_update 2>/dev/null || true ``` @@ -719,521 +558,51 @@ The graph is the map. Your job after the pipeline is to be the guide. --- -## For --update (incremental re-extraction) +## Interpreter guard for subcommands -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" fi ``` +## For --update and --cluster-only -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction) - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None -new_extract = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` -Clean up after: `rm -f .graphify_old.json` - ---- - -## For --cluster-only - -Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. --- ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" +graphify query "" ``` -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - -Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. --- -## For /graphify path +## For /graphify add and --watch -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. --- -## For /graphify explain +## For the commit hook and native CLAUDE.md integration -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- - -## For /graphify add - -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text - ---- - -## For --watch - -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. - ---- - -## For git commit hook - -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - -If a post-commit hook already exists, graphify appends to it rather than replacing it. - ---- - -## For native CLAUDE.md integration - -Run once per project to make graphify always-on in Claude Code sessions: - -```bash -graphify claude install -``` - -This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. - -```bash -graphify claude uninstall # remove the section -``` +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. --- diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index c0969e72..b9a70f06 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -1,6 +1,6 @@ --- name: graphify -description: "any input (code, docs, papers, images) → knowledge graph → clustered communities → HTML + JSON + audit report. Use when user asks any question about a codebase, project content, architecture, or file relationships — especially if graphify-out/ exists. Provides persistent graph with god nodes, community detection, and BFS/DFS query tools." +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." trigger: /graphify --- @@ -13,8 +13,13 @@ Turn any folder of files into a navigable knowledge graph with community detecti ``` /graphify # full pipeline on current directory → Obsidian vault /graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON /graphify --html # (HTML is generated by default - this flag is a no-op) @@ -24,6 +29,8 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j /graphify --mcp # start MCP stdio server for agent access /graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) /graphify add # fetch URL, save to ./raw, update graph /graphify add --author "Name" # tag who wrote it /graphify add --contributor "Name" # tag who added it to the corpus @@ -36,29 +43,24 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes → one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. ## What You Must Do When Invoked If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + If no path was given, use `.` (current directory). Do not ask the user for a path. -If `graphify-out/` exists, use `graphify query`, `graphify explain`, or `graphify path` for orientation before broad grep, rg, or multi-file reads. Dirty `graphify-out/` artifacts are expected after hooks or incremental updates; dirty graph files are not a reason to skip Graphify. Only skip Graphify if the task is specifically about stale or incorrect graph output, or the user explicitly says not to use it. +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. Follow these steps in order. Do not skip steps. +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + ### Step 1 - Ensure graphify is installed ```bash @@ -93,6 +95,8 @@ fi # Write interpreter path for all subsequent steps (persists across invocations) mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -107,8 +111,8 @@ import json from graphify.detect import detect from pathlib import Path result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) -" > .graphify_detect.json +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -127,58 +131,31 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. - Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. -### Step 2.5 - Transcribe video / audio files (only if video files detected) +### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. ### Step 3 - Extract entities and relationships **Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** @@ -196,16 +173,16 @@ from pathlib import Path import json code_files = [] -detect = json.loads(Path('.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files) - Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: - Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') " ``` @@ -217,7 +194,7 @@ else: **MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `.graphify_detect.json` +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` - Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) - Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) - Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" @@ -232,23 +209,23 @@ import json from graphify.cache import check_semantic_cache from pathlib import Path -detect = json.loads(Path('.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) all_files = [f for files in detect['files'].values() for f in files] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) if cached_nodes or cached_edges or cached_hyperedges: - Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " ``` -Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. **Step B2 - Dispatch ALL subagents in a single message (Codex)** @@ -256,74 +233,22 @@ Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. > Requires `multi_agent = true` under `[features]` in `~/.codex/config.toml`. > If `spawn_agent` is unavailable, tell the user to add that config and restart Codex. -Call `spawn_agent` once per chunk — ALL in the same response so they run in parallel. Build the message by wrapping the extraction prompt below in task-delegation framing: +Call `spawn_agent` once per chunk — ALL in the same response so they run in parallel. Build the message by wrapping the extraction prompt in task-delegation framing: ``` -spawn_agent(agent_type="worker", message="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt below, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") +spawn_agent(agent_type="worker", message="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") ``` -After all agents are dispatched, collect results sequentially: +After all agents are dispatched, collect results sequentially in memory: ``` result = wait_agent(handle); close_agent(handle) # repeat per handle ``` -Parse each result as JSON. Accumulate nodes/edges/hyperedges across all results and write to `.graphify_semantic_new.json`. +Parse each result as JSON. Accumulate nodes/edges/hyperedges across all results and write to `graphify-out/.graphify_semantic_new.json`. Codex collects in memory, so there are no per-chunk files on disk; the disk-based success checks in Step B3 do not apply — a chunk that returns invalid JSON is the failure signal instead. -The extraction prompt each subagent receives (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE): +Subagent prompt template: -``` -You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. -Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. - -Files (chunk CHUNK_NUM of TOTAL_CHUNKS): -FILE_LIST - -Rules: -- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") -- INFERRED: reasonable inference (shared data structure, implied dependency) -- AMBIGUOUS: uncertain - flag for review, do not omit - -Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). - Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant named node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use the closest existing `file_type` (`document` for prose, `code` for code-derived concepts). Do NOT invent file_types like `concept` or `rationale` — valid values are only `code|document|paper|image`. -Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. -Image files: use vision to understand what the image IS - do not just OCR. - UI screenshot: layout patterns, design decisions, key elements, purpose. - Chart: metric, trend/insight, data source. - Tweet/post: claim as node, author, concepts mentioned. - Diagram: components and connections. - Research figure: what it demonstrates, method, result. - Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. - -DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, - shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - -Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: -- Two functions that both validate user input but never call each other -- A class in code and a concept in a paper that describe the same algorithm -- Two error types that handle the same failure mode differently -Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. - -Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: -- All classes that implement a common protocol or interface -- All functions in an authentication flow (even if they don't all call each other) -- All concepts from a paper section that form one coherent idea -Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. - -If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, - contributor onto every node from that file. - -confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: -- EXTRACTED edges: confidence_score = 1.0 always -- INFERRED edges: reason about each edge individually. - Direct structural evidence (shared data structure, clear dependency): 0.8-0.9. - Reasonable inference with some uncertainty: 0.6-0.7. - Weak or speculative: 0.4-0.5. Most edges should be 0.6-0.9, not 0.5. -- AMBIGUOUS edges: 0.1-0.3 - -Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} -``` +See `references/extraction-spec.md` for the compact subagent prompt (rules, node-ID format, confidence rubric, hyperedge and vision rules, JSON schema). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each agent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted, and have it return the JSON inline. **Step B3 - Collect, cache, and merge** @@ -340,17 +265,12 @@ Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent cal $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path -from graphify.semantic_cleanup import load_validated_semantic_fragment, sanitize_semantic_fragment chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: - d, errors = load_validated_semantic_fragment(Path(c)) - if errors: - print(f'Skipping invalid chunk {c}: ' + '; '.join(errors[:3])) - continue - d = sanitize_semantic_fragment(d) + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) @@ -359,7 +279,7 @@ for c in chunks: Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') " ``` @@ -371,21 +291,20 @@ import json from graphify.cache import save_semantic_cache from pathlib import Path -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) print(f'Cached {saved} files') " ``` -Merge cached + new results into `.graphify_semantic.json`: +Merge cached + new results into `graphify-out/.graphify_semantic.json`: ```bash $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path -from graphify.semantic_cleanup import sanitize_semantic_fragment -cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) @@ -404,12 +323,11 @@ merged = { 'input_tokens': new.get('input_tokens', 0), 'output_tokens': new.get('output_tokens', 0), } -merged = sanitize_semantic_fragment(merged) -Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction @@ -417,10 +335,9 @@ Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphi $(cat graphify-out/.graphify_python) -c " import sys, json from pathlib import Path -from graphify.semantic_cleanup import sanitize_semantic_fragment -ast = json.loads(Path('.graphify_ast.json').read_text()) -sem = json.loads(Path('.graphify_semantic.json').read_text()) +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} @@ -439,8 +356,7 @@ merged = { 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } -merged = sanitize_semantic_fragment(merged) -Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") total = len(merged_nodes) edges = len(merged_edges) print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') @@ -449,6 +365,8 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + ```bash mkdir -p graphify-out $(cat graphify-out/.graphify_python) -c " @@ -460,8 +378,8 @@ from graphify.report import generate from graphify.export import to_json from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = cluster(G) @@ -473,8 +391,8 @@ labels = {cid: 'Community ' + str(cid) for cid in communities} # Placeholder questions - regenerated with real labels in Step 5 questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") to_json(G, communities, 'graphify-out/graph.json') analysis = { @@ -484,7 +402,7 @@ analysis = { 'surprises': surprises, 'questions': questions, } -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") if G.number_of_nodes() == 0: print('ERROR: Graph is empty - extraction produced no nodes.') print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') @@ -499,7 +417,7 @@ Replace INPUT_PATH with the actual path. ### Step 5 - Label communities -Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). Then regenerate the report and save the labels for the visualizer: @@ -512,9 +430,9 @@ from graphify.analyze import god_nodes, surprising_connections, suggest_question from graphify.report import generate from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} @@ -527,9 +445,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') " ``` @@ -543,178 +461,23 @@ Replace INPUT_PATH with the actual path. If `--obsidian` was given: +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -if G.number_of_nodes() > 5000: - print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` -### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('.graphify_extract.json').read_text())) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - -### Step 7b - SVG export (only if --svg flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` - -### Step 7c - GraphML export (only if --graphml flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` - -### Step 7d - MCP server (only if --mcp flag) - -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` - -### Step 8 - Token reduction benchmark (only if total_words > 5000) - -If `total_words` from `.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. --- @@ -728,17 +491,19 @@ from datetime import datetime, timezone from graphify.detect import save_manifest # Save manifest for --update -detect = json.loads(Path('.graphify_detect.json').read_text()) -save_manifest(detect['files']) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) # Update cumulative cost tracker -extract = json.loads(Path('.graphify_extract.json').read_text()) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) cost_path = Path('graphify-out/cost.json') if cost_path.exists(): - cost = json.loads(cost_path.read_text()) + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) else: cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} @@ -750,12 +515,12 @@ cost['runs'].append({ }) cost['total_input_tokens'] += input_tok cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_chunk_*.json +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json rm -f graphify-out/.needs_update 2>/dev/null || true ``` @@ -790,521 +555,51 @@ The graph is the map. Your job after the pipeline is to be the guide. --- -## For --update (incremental re-extraction) +## Interpreter guard for subcommands -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" fi ``` +## For --update and --cluster-only -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction) - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None -new_extract = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` -Clean up after: `rm -f .graphify_old.json` - ---- - -## For --cluster-only - -Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. --- ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" +graphify query "" ``` -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - -Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. --- -## For /graphify path +## For /graphify add and --watch -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. --- -## For /graphify explain +## For the commit hook and native CLAUDE.md integration -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- - -## For /graphify add - -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text - ---- - -## For --watch - -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. - ---- - -## For git commit hook - -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - -If a post-commit hook already exists, graphify appends to it rather than replacing it. - ---- - -## For native CLAUDE.md integration - -Run once per project to make graphify always-on in Claude Code sessions: - -```bash -graphify claude install -``` - -This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. - -```bash -graphify claude uninstall # remove the section -``` +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. --- diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index e1650dde..bad0a0d5 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -1,6 +1,6 @@ --- name: graphify -description: "any input (code, docs, papers, images) → knowledge graph → clustered communities → HTML + JSON + audit report. Use when user asks any question about a codebase, project content, architecture, or file relationships — especially if graphify-out/ exists. Provides persistent graph with god nodes, community detection, and BFS/DFS query tools." +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." trigger: /graphify --- @@ -13,8 +13,13 @@ Turn any folder of files into a navigable knowledge graph with community detecti ``` /graphify # full pipeline on current directory → Obsidian vault /graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON /graphify --html # (HTML is generated by default - this flag is a no-op) @@ -38,27 +43,24 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes → one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. ## What You Must Do When Invoked If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + If no path was given, use `.` (current directory). Do not ask the user for a path. +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + Follow these steps in order. Do not skip steps. +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + ### Step 1 - Ensure graphify is installed ```bash @@ -93,6 +95,8 @@ fi # Write interpreter path for all subsequent steps (persists across invocations) mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -107,7 +111,7 @@ import json from graphify.detect import detect from pathlib import Path result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) +print(json.dumps(result, ensure_ascii=False)) " > graphify-out/.graphify_detect.json ``` @@ -127,58 +131,31 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. - Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. -### Step 2.5 - Transcribe video / audio files (only if video files detected) +### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. ### Step 3 - Extract entities and relationships **Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** @@ -196,16 +173,16 @@ from pathlib import Path import json code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2)) + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') " ``` @@ -232,14 +209,14 @@ import json from graphify.cache import check_semantic_cache from pathlib import Path -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) all_files = [f for files in detect['files'].values() for f in files] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached)) + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " ``` @@ -254,69 +231,27 @@ Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-2 Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. + Concrete example for 3 chunks: ``` -[Agent tool call 1: files 1-15] -[Agent tool call 2: files 16-30] -[Agent tool call 3: files 31-45] +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] ``` All three in one message. Not three separate messages. -Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE): +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" ``` -You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. -Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. -Files (chunk CHUNK_NUM of TOTAL_CHUNKS): -FILE_LIST +Subagent prompt template: -Rules: -- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") -- INFERRED: reasonable inference (shared data structure, implied dependency) -- AMBIGUOUS: uncertain - flag for review, do not omit - -Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). - Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). Do NOT invent file_types like `concept` — valid values are only `code|document|paper|image|rationale`. -Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. -Image files: use vision to understand what the image IS - do not just OCR. - UI screenshot: layout patterns, design decisions, key elements, purpose. - Chart: metric, trend/insight, data source. - Tweet/post: claim as node, author, concepts mentioned. - Diagram: components and connections. - Research figure: what it demonstrates, method, result. - Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. - -DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, - shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - -Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: -- Two functions that both validate user input but never call each other -- A class in code and a concept in a paper that describe the same algorithm -- Two error types that handle the same failure mode differently -Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. - -Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: -- All classes that implement a common protocol or interface -- All functions in an authentication flow (even if they don't all call each other) -- All concepts from a paper section that form one coherent idea -Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. - -If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, - contributor onto every node from that file. - -confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: -- EXTRACTED edges: confidence_score = 1.0 always -- INFERRED edges: reason about each edge individually. - Direct structural evidence (shared data structure, clear dependency): 0.8-0.9. - Reasonable inference with some uncertainty: 0.6-0.7. - Weak or speculative: 0.4-0.5. Most edges should be 0.6-0.9, not 0.5. -- AMBIGUOUS edges: 0.1-0.3 - -Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} -``` +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. **Step B3 - Collect, cache, and merge** @@ -338,7 +273,7 @@ chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: - d = json.loads(Path(c).read_text()) + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) @@ -347,7 +282,7 @@ for c in chunks: Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') " ``` @@ -359,7 +294,7 @@ import json from graphify.cache import save_semantic_cache from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) print(f'Cached {saved} files') " @@ -371,8 +306,8 @@ $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text()) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) @@ -391,7 +326,7 @@ merged = { 'input_tokens': new.get('input_tokens', 0), 'output_tokens': new.get('output_tokens', 0), } -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` @@ -404,8 +339,8 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from pathlib import Path -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text()) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text()) +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} @@ -424,7 +359,7 @@ merged = { 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") total = len(merged_nodes) edges = len(merged_edges) print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') @@ -433,6 +368,8 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + ```bash mkdir -p graphify-out $(cat graphify-out/.graphify_python) -c " @@ -444,8 +381,8 @@ from graphify.report import generate from graphify.export import to_json from pathlib import Path -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = cluster(G) @@ -457,8 +394,8 @@ labels = {cid: 'Community ' + str(cid) for cid in communities} # Placeholder questions - regenerated with real labels in Step 5 questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") to_json(G, communities, 'graphify-out/graph.json') analysis = { @@ -468,7 +405,7 @@ analysis = { 'surprises': surprises, 'questions': questions, } -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") if G.number_of_nodes() == 0: print('ERROR: Graph is empty - extraction produced no nodes.') print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') @@ -496,9 +433,9 @@ from graphify.analyze import god_nodes, surprising_connections, suggest_question from graphify.report import generate from pathlib import Path -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} @@ -511,9 +448,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') " ``` @@ -527,182 +464,23 @@ Replace INPUT_PATH with the actual path. If `--obsidian` was given: -- If `--obsidian-dir ` was also given, use that path as the vault directory. Otherwise default to `graphify-out/obsidian`. +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -obsidian_dir = 'OBSIDIAN_DIR' # replace with --obsidian-dir value, or 'graphify-out/obsidian' if not given - -n = to_obsidian(G, communities, obsidian_dir, community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in {obsidian_dir}/') - -to_canvas(G, communities, f'{obsidian_dir}/graph.canvas', community_labels=labels or None) -print(f'Canvas: {obsidian_dir}/graph.canvas - open in Obsidian for structured community layout') -print() -print(f'Open {obsidian_dir}/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -if G.number_of_nodes() > 5000: - print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` -### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text())) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - -### Step 7b - SVG export (only if --svg flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` - -### Step 7c - GraphML export (only if --graphml flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` - -### Step 7d - MCP server (only if --mcp flag) - -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` - -### Step 8 - Token reduction benchmark (only if total_words > 5000) - -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. --- @@ -716,17 +494,19 @@ from datetime import datetime, timezone from graphify.detect import save_manifest # Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -save_manifest(detect['files']) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) cost_path = Path('graphify-out/cost.json') if cost_path.exists(): - cost = json.loads(cost_path.read_text()) + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) else: cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} @@ -738,12 +518,12 @@ cost['runs'].append({ }) cost['total_input_tokens'] += input_tok cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json rm -f graphify-out/.needs_update 2>/dev/null || true ``` @@ -792,536 +572,37 @@ if [ ! -f graphify-out/.graphify_python ]; then PYTHON="python3" fi mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" fi ``` -## For --update (incremental re-extraction) +## For --update and --cluster-only -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction) - -# Prune nodes from deleted files -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text()) -deleted = set(incremental.get('deleted_files', [])) -if deleted: - to_remove = [n for n, d in G_existing.nodes(data=True) if d.get('source_file') in deleted] - G_existing.remove_nodes_from(to_remove) - if to_remove: - print(f'Pruned {len(to_remove)} ghost node(s) from {len(deleted)} deleted file(s).') - else: - print(f'{len(deleted)} file(s) deleted since last run — no ghost nodes in graph, already clean.') - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text()) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` - ---- - -## For --cluster-only - -Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. --- ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" +graphify query "" ``` -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - -Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. --- -## For /graphify path +## For /graphify add and --watch -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. --- -## For /graphify explain +## For the commit hook and native CLAUDE.md integration -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- - -## For /graphify add - -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text - ---- - -## For --watch - -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. - ---- - -## For git commit hook - -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - -If a post-commit hook already exists, graphify appends to it rather than replacing it. - ---- - -## For native CLAUDE.md integration - -Run once per project to make graphify always-on in Claude Code sessions: - -```bash -graphify claude install -``` - -This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. - -```bash -graphify claude uninstall # remove the section -``` +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. --- diff --git a/graphify/skill-devin.md b/graphify/skill-devin.md index ad99f54a..7013159a 100644 --- a/graphify/skill-devin.md +++ b/graphify/skill-devin.md @@ -1,6 +1,6 @@ --- name: graphify -description: "any input (code, docs, papers, images, video) → knowledge graph → clustered communities → HTML + JSON + GRAPH_REPORT.md. Use when user asks any question about a codebase, project content, architecture, or file relationships — especially if graphify-out/ exists. Provides persistent graph with god nodes, community detection, and BFS/DFS query tools." +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." argument-hint: "[path|query|subcommand]" model: sonnet allowed-tools: @@ -283,7 +283,7 @@ Extraction rules: - INFERRED: reasonable inference (shared structure, implied dependency) - AMBIGUOUS: uncertain — flag it, do not omit - Code files: extract semantic edges AST cannot find (design patterns, protocol conformance). Do not re-extract imports. -- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms). Do NOT invent file_types like `concept`. When adding `calls` edges: source is caller, target is callee. +- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. When adding `calls` edges: source is caller, target is callee. - Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. Chart: metric, trend/insight, data source. @@ -311,7 +311,7 @@ Extraction rules: - AMBIGUOUS edges: 0.1-0.3 Schema: -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} Files: FILE_LIST diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 0f83c3d5..6c965c67 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -1,6 +1,6 @@ --- name: graphify -description: "any input (code, docs, papers, images) → knowledge graph → clustered communities → HTML + JSON + audit report. Use when user asks any question about a codebase, project content, architecture, or file relationships — especially if graphify-out/ exists. Provides persistent graph with god nodes, community detection, and BFS/DFS query tools." +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." trigger: /graphify --- @@ -13,8 +13,13 @@ Turn any folder of files into a navigable knowledge graph with community detecti ``` /graphify # full pipeline on current directory → Obsidian vault /graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON /graphify --html # (HTML is generated by default - this flag is a no-op) @@ -24,6 +29,8 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j /graphify --mcp # start MCP stdio server for agent access /graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) /graphify add # fetch URL, save to ./raw, update graph /graphify add --author "Name" # tag who wrote it /graphify add --contributor "Name" # tag who added it to the corpus @@ -36,27 +43,24 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes → one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. ## What You Must Do When Invoked If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + If no path was given, use `.` (current directory). Do not ask the user for a path. +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + Follow these steps in order. Do not skip steps. +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + ### Step 1 - Ensure graphify is installed ```bash @@ -91,6 +95,8 @@ fi # Write interpreter path for all subsequent steps (persists across invocations) mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -105,8 +111,8 @@ import json from graphify.detect import detect from pathlib import Path result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) -" > .graphify_detect.json +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -125,58 +131,31 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. - Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. -### Step 2.5 - Transcribe video / audio files (only if video files detected) +### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. ### Step 3 - Extract entities and relationships **Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** @@ -194,16 +173,16 @@ from pathlib import Path import json code_files = [] -detect = json.loads(Path('.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files) - Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: - Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') " ``` @@ -215,7 +194,7 @@ else: **MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `.graphify_detect.json` +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` - Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) - Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) - Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" @@ -230,94 +209,46 @@ import json from graphify.cache import check_semantic_cache from pathlib import Path -detect = json.loads(Path('.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) all_files = [f for files in detect['files'].values() for f in files] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) if cached_nodes or cached_edges or cached_hyperedges: - Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " ``` -Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. -**Step B2 - Dispatch ALL subagents in a single message (Factory Droid)** +**Step B2 - Dispatch ALL subagents in a single message** -> **Factory Droid platform:** Uses the `Task` tool for parallel subagent dispatch. +> Uses the `Task` tool for parallel subagent dispatch. > Call `Task` once per chunk — ALL in the same response so they run in parallel. -Call `Task` once per chunk — ALL in the same response so they run in parallel. Pass the extraction prompt as the task description: +Pass the extraction prompt as the task description: ``` -Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt below, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") +Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") ``` -Collect results as each Task completes. Parse each result as JSON. - -Parse each result as JSON. Accumulate nodes/edges/hyperedges across all results and write to `.graphify_semantic_new.json`. - -The extraction prompt each subagent receives (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE): +Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" ``` -You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. -Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. -Files (chunk CHUNK_NUM of TOTAL_CHUNKS): -FILE_LIST +Subagent prompt template: -Rules: -- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") -- INFERRED: reasonable inference (shared data structure, implied dependency) -- AMBIGUOUS: uncertain - flag for review, do not omit - -Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). - Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). Do NOT invent file_types like `concept` — valid values are only `code|document|paper|image|rationale`. -Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. -Image files: use vision to understand what the image IS - do not just OCR. - UI screenshot: layout patterns, design decisions, key elements, purpose. - Chart: metric, trend/insight, data source. - Tweet/post: claim as node, author, concepts mentioned. - Diagram: components and connections. - Research figure: what it demonstrates, method, result. - Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. - -DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, - shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - -Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: -- Two functions that both validate user input but never call each other -- A class in code and a concept in a paper that describe the same algorithm -- Two error types that handle the same failure mode differently -Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. - -Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: -- All classes that implement a common protocol or interface -- All functions in an authentication flow (even if they don't all call each other) -- All concepts from a paper section that form one coherent idea -Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. - -If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, - contributor onto every node from that file. - -confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: -- EXTRACTED edges: confidence_score = 1.0 always -- INFERRED edges: reason about each edge individually. - Direct structural evidence (shared data structure, clear dependency): 0.8-0.9. - Reasonable inference with some uncertainty: 0.6-0.7. - Weak or speculative: 0.4-0.5. Most edges should be 0.6-0.9, not 0.5. -- AMBIGUOUS edges: 0.1-0.3 - -Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} -``` +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. **Step B3 - Collect, cache, and merge** @@ -339,7 +270,7 @@ chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: - d = json.loads(Path(c).read_text()) + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) @@ -348,7 +279,7 @@ for c in chunks: Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') " ``` @@ -360,20 +291,20 @@ import json from graphify.cache import save_semantic_cache from pathlib import Path -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) print(f'Cached {saved} files') " ``` -Merge cached + new results into `.graphify_semantic.json`: +Merge cached + new results into `graphify-out/.graphify_semantic.json`: ```bash $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path -cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) @@ -392,11 +323,11 @@ merged = { 'input_tokens': new.get('input_tokens', 0), 'output_tokens': new.get('output_tokens', 0), } -Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction @@ -405,8 +336,8 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from pathlib import Path -ast = json.loads(Path('.graphify_ast.json').read_text()) -sem = json.loads(Path('.graphify_semantic.json').read_text()) +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} @@ -425,7 +356,7 @@ merged = { 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } -Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") total = len(merged_nodes) edges = len(merged_edges) print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') @@ -434,6 +365,8 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + ```bash mkdir -p graphify-out $(cat graphify-out/.graphify_python) -c " @@ -445,8 +378,8 @@ from graphify.report import generate from graphify.export import to_json from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = cluster(G) @@ -458,8 +391,8 @@ labels = {cid: 'Community ' + str(cid) for cid in communities} # Placeholder questions - regenerated with real labels in Step 5 questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") to_json(G, communities, 'graphify-out/graph.json') analysis = { @@ -469,7 +402,7 @@ analysis = { 'surprises': surprises, 'questions': questions, } -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") if G.number_of_nodes() == 0: print('ERROR: Graph is empty - extraction produced no nodes.') print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') @@ -484,7 +417,7 @@ Replace INPUT_PATH with the actual path. ### Step 5 - Label communities -Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). Then regenerate the report and save the labels for the visualizer: @@ -497,9 +430,9 @@ from graphify.analyze import god_nodes, surprising_connections, suggest_question from graphify.report import generate from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} @@ -512,9 +445,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') " ``` @@ -528,178 +461,23 @@ Replace INPUT_PATH with the actual path. If `--obsidian` was given: +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -if G.number_of_nodes() > 5000: - print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` -### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('.graphify_extract.json').read_text())) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - -### Step 7b - SVG export (only if --svg flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` - -### Step 7c - GraphML export (only if --graphml flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` - -### Step 7d - MCP server (only if --mcp flag) - -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` - -### Step 8 - Token reduction benchmark (only if total_words > 5000) - -If `total_words` from `.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. --- @@ -713,17 +491,19 @@ from datetime import datetime, timezone from graphify.detect import save_manifest # Save manifest for --update -detect = json.loads(Path('.graphify_detect.json').read_text()) -save_manifest(detect['files']) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) # Update cumulative cost tracker -extract = json.loads(Path('.graphify_extract.json').read_text()) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) cost_path = Path('graphify-out/cost.json') if cost_path.exists(): - cost = json.loads(cost_path.read_text()) + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) else: cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} @@ -735,12 +515,12 @@ cost['runs'].append({ }) cost['total_input_tokens'] += input_tok cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_chunk_*.json +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json rm -f graphify-out/.needs_update 2>/dev/null || true ``` @@ -775,521 +555,51 @@ The graph is the map. Your job after the pipeline is to be the guide. --- -## For --update (incremental re-extraction) +## Interpreter guard for subcommands -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" fi ``` +## For --update and --cluster-only -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction) - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None -new_extract = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` -Clean up after: `rm -f .graphify_old.json` - ---- - -## For --cluster-only - -Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. --- ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" +graphify query "" ``` -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - -Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. --- -## For /graphify path +## For /graphify add and --watch -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. --- -## For /graphify explain +## For the commit hook and native CLAUDE.md integration -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- - -## For /graphify add - -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text - ---- - -## For --watch - -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. - ---- - -## For git commit hook - -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - -If a post-commit hook already exists, graphify appends to it rather than replacing it. - ---- - -## For native CLAUDE.md integration - -Run once per project to make graphify always-on in Claude Code sessions: - -```bash -graphify claude install -``` - -This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. - -```bash -graphify claude uninstall # remove the section -``` +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. --- diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index c3b76761..8bc6d536 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -1,55 +1,107 @@ --- name: graphify -description: any input (code, docs, papers, images) -> knowledge graph -> clustered communities -> HTML + JSON + audit report +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." trigger: /graphify --- # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language `GRAPH_REPORT.md`. +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. ## Usage -```text -/graphify -/graphify -/graphify --mode deep -/graphify --update -/graphify --cluster-only -/graphify --no-viz -/graphify add -/graphify query "" -/graphify path "AuthModule" "Database" -/graphify explain "SwinTransformer" ``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. ## What You Must Do When Invoked -If no path was given, use `.`. Do not ask the user for a path. +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. Follow these steps in order. Do not skip steps. +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + ### Step 1 - Ensure graphify is installed ```bash -# Detect the correct Python interpreter (handles pipx, venv, system installs) +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) -if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in - *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; - esac -else - PYTHON="python3" +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi -"$PYTHON" -c "import graphify" 2>/dev/null || "$PYTHON" -m pip install graphifyy -q 2>/dev/null || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) mkdir -p graphify-out -"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. -**In every later bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to reuse the correct interpreter.** +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** ### Step 2 - Detect files @@ -58,98 +110,98 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.detect import detect from pathlib import Path - result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) +print(json.dumps(result, ensure_ascii=False)) " > graphify-out/.graphify_detect.json ``` -Replace `INPUT_PATH` with the actual path the user provided. Do not dump the JSON to the user. Read it silently and present a clean summary instead: +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: -```text +``` Corpus: X files · ~Y words - code: N files - docs: N files - papers: N files + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) images: N files - video: N files + video: N files (.mp4 .mp3 ...) ``` -Omit any category with 0 files. +Omit any category with 0 files from the summary. Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. -- If `total_files` is 0: stop with `No supported files found in [path].` -- If `skipped_sensitive` is non-empty: mention the count skipped, not the filenames. -- If `total_words > 2_000_000` or `total_files > 200`: show the warning and ask which subfolder to run on. -- Otherwise continue directly to Step 2.5 if video files were detected, or Step 3 if not. +### Step 2.5 - Video and audio (only if video files detected) -### Step 2.5 - Transcribe video or audio files (only if video files were detected) - -Skip this step entirely if detection found zero `video` files. - -Transcribe the files first, then treat the transcripts as docs during semantic extraction. - -Write a one-sentence Whisper prompt yourself from the top graph labels. If the corpus only contains video, use `Use proper punctuation and paragraph breaks.` - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: - -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before semantic extraction -- Print `Transcribed N video file(s) -> treating as docs` -- If a file fails, warn and continue with the rest +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. ### Step 3 - Extract entities and relationships -Track whether `--mode deep` was given. Pass that through to every semantic extraction task. +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. -This step has two parts: deterministic AST extraction for code and semantic extraction for docs, papers, and images. +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. #### Part A - Structural extraction for code files +For any code files detected, run AST extraction in parallel with Part B subagents: + ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: result = extract(code_files, cache_root=Path('.')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2)) - print('AST: {} nodes, {} edges'.format(len(result['nodes']), len(result['edges']))) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes': [], 'edges': [], 'input_tokens': 0, 'output_tokens': 0})) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') " ``` -#### Part B - Semantic extraction for docs, papers, and images +#### Part B - Semantic extraction (parallel subagents) -If detection found zero docs, papers, and images, skip this part and go straight to Part C. +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. -**You must use the native `Task` tool here.** Reading files sequentially yourself is too slow and defeats the purpose of the Kilo integration. +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** -Before dispatching tasks, check the semantic cache first: +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: ```bash $(cat graphify-out/.graphify_python) -c " @@ -157,196 +209,282 @@ import json from graphify.cache import check_semantic_cache from pathlib import Path -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) all_files = [f for files in detect['files'].values() for f in files] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached)) -print(f'Cache: {len(all_files) - len(uncached)} files hit, {len(uncached)} files need extraction') + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " ``` -If every file is cached, skip directly to Part C. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. -Split `graphify-out/.graphify_uncached.txt` into chunks of 20-25 files, grouping related directories together. Each image should get its own chunk. +**Step B1 - Split into chunks** -Dispatch **all** chunk tasks in the same response so they run in parallel. Always use `subagent_type="general"`. +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. -Each task should receive a prompt like this, with the placeholders replaced: +**Step B2 - Dispatch ALL subagents in a single message** -```text -You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. -Output ONLY valid JSON matching the schema below. +Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. -Files (chunk CHUNK_NUM of TOTAL_CHUNKS): -FILE_LIST +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. -Rules: -- EXTRACTED: relationship explicit in source -- INFERRED: reasonable inference -- AMBIGUOUS: uncertain, flag it instead of omitting it -- Code files: focus on semantic edges AST cannot find -- Doc and paper files: extract concepts, citations, and rationale nodes -- Image files: use vision to understand the artifact, not just OCR -- If DEEP_MODE is true, be more aggressive with INFERRED edges -- Include `confidence_score` on every edge +Concrete example for 3 chunks: +``` +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] +``` +All three in one message. Not three separate messages. -Write the result to `graphify-out/.graphify_chunk_CHUNK_NUM.json` and also return the same JSON. +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -Schema: -{"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" ``` -After all tasks finish, merge them into `graphify-out/.graphify_semantic_new.json`. Track the chunk filenames yourself and substitute them below. +Subagent prompt template: +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, glob from pathlib import Path -chunk_files = CHUNK_FILES -combined = {'nodes': [], 'edges': [], 'hyperedges': [], 'input_tokens': 0, 'output_tokens': 0} -failures = 0 - -for chunk_file in chunk_files: - path = Path(chunk_file) - if not path.exists(): - failures += 1 - continue - try: - data = json.loads(path.read_text()) - except Exception: - failures += 1 - continue - combined['nodes'].extend(data.get('nodes', [])) - combined['edges'].extend(data.get('edges', [])) - combined['hyperedges'].extend(data.get('hyperedges', [])) - combined['input_tokens'] += data.get('input_tokens', 0) - combined['output_tokens'] += data.get('output_tokens', 0) - -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps(combined, indent=2)) -print('Semantic extraction: {} nodes, {} edges, {} failed chunk(s)'.format(len(combined['nodes']), len(combined['edges']), failures)) +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') " ``` -If more than half the chunk tasks fail, stop and report the failure instead of silently continuing. - -Cache the new semantic results and merge them with cached data: - +Save new results to cache: ```bash $(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes': [], 'edges': [], 'hyperedges': []} -save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text()) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes': [], 'edges': [], 'hyperedges': []} +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path -all_nodes = cached.get('nodes', []) + new.get('nodes', []) -all_edges = cached.get('edges', []) + new.get('edges', []) +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) - seen = set() -deduped_nodes = [] -for node in all_nodes: - node_id = node.get('id') - if node_id and node_id not in seen: - seen.add(node_id) - deduped_nodes.append(node) +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) merged = { - 'nodes': deduped_nodes, + 'nodes': deduped, 'edges': all_edges, 'hyperedges': all_hyperedges, 'input_tokens': new.get('input_tokens', 0), 'output_tokens': new.get('output_tokens', 0), } -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) -print(f'Semantic merge complete - {len(deduped_nodes)} nodes, {len(all_edges)} edges') +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` -#### Part C - Merge AST and semantic extraction +#### Part C - Merge AST + semantic into final extraction ```bash $(cat graphify-out/.graphify_python) -c " -import json +import sys, json from pathlib import Path -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text()) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text()) if Path('graphify-out/.graphify_semantic.json').exists() else {'nodes': [], 'edges': [], 'hyperedges': [], 'input_tokens': 0, 'output_tokens': 0} +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) -for node in sem.get('nodes', []): - if node['id'] not in seen: - merged_nodes.append(node) - seen.add(node['id']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) merged = { 'nodes': merged_nodes, - 'edges': ast['edges'] + sem.get('edges', []), - 'hyperedges': sem.get('hyperedges', []), + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2)) -print('Merged extraction: {} nodes, {} edges'.format(len(merged_nodes), len(merged['edges']))) +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') " ``` -### Step 4 - Build the graph and generate outputs +### Step 4 - Build graph, cluster, analyze, generate outputs -Use generic labels like `Community 0`, `Community 1`, and so on. Do not stop to ask for manual labels. +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. ```bash +mkdir -p graphify-out $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path +import sys, json from graphify.build import build_from_json from graphify.cluster import cluster, score_all from graphify.analyze import god_nodes, surprising_connections, suggest_questions from graphify.report import generate -from graphify.export import to_json, to_html +from graphify.export import to_json +from pathlib import Path -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = cluster(G) cohesion = score_all(G, communities) -labels = {cid: f'Community {cid}' for cid in communities} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} gods = god_nodes(G) surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 questions = suggest_questions(G, communities, labels) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding='utf-8') +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") to_json(G, communities, 'graphify-out/graph.json') -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps({'questions': questions, 'gods': gods, 'surprises': surprises, 'labels': labels}, indent=2), encoding='utf-8') +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") if G.number_of_nodes() == 0: print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') raise SystemExit(1) - -if 'NO_VIZ' != 'true' and G.number_of_nodes() <= 5000: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels) - print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') " ``` -Replace `INPUT_PATH` with the actual path. Replace `NO_VIZ` with `true` if `--no-viz` was given, otherwise `false`. +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. -If the graph is empty, stop and report the failure. +Replace INPUT_PATH with the actual path. -### Step 5 - Save manifest, clean up, and report +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report ```bash $(cat graphify-out/.graphify_python) -c " @@ -355,60 +493,132 @@ from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -save_manifest(detect['files']) +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) cost_path = Path('graphify-out/cost.json') -cost = json.loads(cost_path.read_text()) if cost_path.exists() else {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} -cost['runs'].append({'date': datetime.now(timezone.utc).isoformat(), 'input_tokens': input_tok, 'output_tokens': output_tok, 'files': detect.get('total_files', 0)}) +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) cost['total_input_tokens'] += input_tok cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2), encoding='utf-8') +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_extract.json +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true ``` -Tell the user where the outputs were written: - -```text +Tell the user (omit the obsidian line unless --obsidian was given): +``` Graph complete. Outputs in PATH_TO_DIR/graphify-out/ graph.html - interactive graph, open in browser GRAPH_REPORT.md - audit report graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) ``` -Then summarize the god nodes, surprising connections, and suggested questions from `GRAPH_REPORT.md` without pasting the entire report. +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi -### Query mode +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. -Before any query-style subcommand, ensure `graphify-out/.graphify_python` exists. If not, recreate it using the Step 1 interpreter detection. +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions -When the user invokes one of these forms, use the CLI directly instead of rebuilding the graph: +Do NOT paste the full report - just those three sections. Keep it concise. -- `/graphify query "..."` -- `/graphify path "A" "B"` -- `/graphify explain "X"` +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: -Use: +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: ```bash -$(cat graphify-out/.graphify_python) -m graphify query "QUESTION" --graph graphify-out/graph.json -$(cat graphify-out/.graphify_python) -m graphify path "SOURCE" "TARGET" --graph graphify-out/graph.json -$(cat graphify-out/.graphify_python) -m graphify explain "NODE" --graph graphify-out/graph.json +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi ``` -Prefer graph-backed answers over guesses, and cite source files when the graph provides them. +## For --update and --cluster-only -### Kilo-specific rules +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: + +```bash +graphify query "" +``` + +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native CLAUDE.md integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. + +--- + +## Kilo-specific rules - Use the native `Task` tool for semantic extraction fan-out. - Launch all chunk tasks in the same response so they run in parallel. - Always use `subagent_type="general"` for extraction chunks. - After modifying code files during the session, run `graphify update .`. + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index ab31b7f9..796ad8ca 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -1,6 +1,6 @@ --- name: graphify -description: Turn any folder of files (code, docs, papers, images, video) into a queryable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. Use when asked to analyze a codebase, understand architecture, map dependencies, or build a knowledge graph. +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." --- # /graphify @@ -12,8 +12,13 @@ Turn any folder of files into a navigable knowledge graph with community detecti ``` /graphify # full pipeline on current directory → Obsidian vault /graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON /graphify --html # (HTML is generated by default - this flag is a no-op) @@ -23,6 +28,8 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j /graphify --mcp # start MCP stdio server for agent access /graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) /graphify add # fetch URL, save to ./raw, update graph /graphify add --author "Name" # tag who wrote it /graphify add --contributor "Name" # tag who added it to the corpus @@ -35,27 +42,24 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes → one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. ## What You Must Do When Invoked If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + If no path was given, use `.` (current directory). Do not ask the user for a path. +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + Follow these steps in order. Do not skip steps. +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + ### Step 1 - Ensure graphify is installed ```bash @@ -90,6 +94,8 @@ fi # Write interpreter path for all subsequent steps (persists across invocations) mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -104,8 +110,8 @@ import json from graphify.detect import detect from pathlib import Path result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) -" > .graphify_detect.json +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -124,58 +130,31 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. - Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. -### Step 2.5 - Transcribe video / audio files (only if video files detected) +### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. ### Step 3 - Extract entities and relationships **Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** @@ -193,16 +172,16 @@ from pathlib import Path import json code_files = [] -detect = json.loads(Path('.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files) - Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: - Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') " ``` @@ -211,9 +190,13 @@ else: **Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. -> **OpenClaw platform:** Multi-agent support is still early on OpenClaw. Extraction runs sequentially — you read and extract each file yourself. This is slower than parallel platforms but fully reliable. +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** -Print: `"Semantic extraction: N files (sequential — OpenClaw)"` +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" **Step B0 - Check extraction cache first** @@ -225,52 +208,59 @@ import json from graphify.cache import check_semantic_cache from pathlib import Path -detect = json.loads(Path('.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) all_files = [f for files in detect['files'].values() for f in files] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) if cached_nodes or cached_edges or cached_hyperedges: - Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " ``` -Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. -**Step B2 - Sequential extraction (OpenClaw)** +**Step B2 - Dispatch ALL subagents in a single message** -Process each file one at a time. For each file: +Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. -1. Read the file contents -2. Extract nodes, edges, and hyperedges applying the same rules: - - EXTRACTED: relationship explicit in source (import, call, citation) - - INFERRED: reasonable inference (shared structure, implied dependency) - - AMBIGUOUS: uncertain — flag it, do not omit - - Code files: semantic edges AST cannot find. Do not re-extract imports. - - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms). Do NOT invent file_types like `concept`. When adding `calls` edges: source is caller, target is callee. - - Image files: use vision — understand what the image IS, not just OCR - - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges - - Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. - - Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. - - confidence_score REQUIRED on every edge: EXTRACTED=1.0, INFERRED=0.6-0.9 (reason individually), AMBIGUOUS=0.1-0.3 -3. Accumulate results across all files +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. -Schema for each file's output: -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +Concrete example for 3 chunks: +``` +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] +``` +All three in one message. Not three separate messages. -After processing all files, write the accumulated result to `.graphify_semantic_new.json`. +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -**Step B3 - Cache and merge** +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` -For the accumulated result: +Subagent prompt template: -If more than half the chunks failed, stop and tell the user. +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash @@ -282,7 +272,7 @@ chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: - d = json.loads(Path(c).read_text()) + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) @@ -291,7 +281,7 @@ for c in chunks: Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') " ``` @@ -303,20 +293,20 @@ import json from graphify.cache import save_semantic_cache from pathlib import Path -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) print(f'Cached {saved} files') " ``` -Merge cached + new results into `.graphify_semantic.json`: +Merge cached + new results into `graphify-out/.graphify_semantic.json`: ```bash $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path -cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) @@ -335,11 +325,11 @@ merged = { 'input_tokens': new.get('input_tokens', 0), 'output_tokens': new.get('output_tokens', 0), } -Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction @@ -348,8 +338,8 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from pathlib import Path -ast = json.loads(Path('.graphify_ast.json').read_text()) -sem = json.loads(Path('.graphify_semantic.json').read_text()) +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} @@ -368,7 +358,7 @@ merged = { 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } -Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") total = len(merged_nodes) edges = len(merged_edges) print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') @@ -377,6 +367,8 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + ```bash mkdir -p graphify-out $(cat graphify-out/.graphify_python) -c " @@ -388,8 +380,8 @@ from graphify.report import generate from graphify.export import to_json from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = cluster(G) @@ -401,8 +393,8 @@ labels = {cid: 'Community ' + str(cid) for cid in communities} # Placeholder questions - regenerated with real labels in Step 5 questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") to_json(G, communities, 'graphify-out/graph.json') analysis = { @@ -412,7 +404,7 @@ analysis = { 'surprises': surprises, 'questions': questions, } -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") if G.number_of_nodes() == 0: print('ERROR: Graph is empty - extraction produced no nodes.') print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') @@ -427,7 +419,7 @@ Replace INPUT_PATH with the actual path. ### Step 5 - Label communities -Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). Then regenerate the report and save the labels for the visualizer: @@ -440,9 +432,9 @@ from graphify.analyze import god_nodes, surprising_connections, suggest_question from graphify.report import generate from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} @@ -455,9 +447,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') " ``` @@ -471,178 +463,23 @@ Replace INPUT_PATH with the actual path. If `--obsidian` was given: +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -if G.number_of_nodes() > 5000: - print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` -### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('.graphify_extract.json').read_text())) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - -### Step 7b - SVG export (only if --svg flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` - -### Step 7c - GraphML export (only if --graphml flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` - -### Step 7d - MCP server (only if --mcp flag) - -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` - -### Step 8 - Token reduction benchmark (only if total_words > 5000) - -If `total_words` from `.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. --- @@ -656,17 +493,19 @@ from datetime import datetime, timezone from graphify.detect import save_manifest # Save manifest for --update -detect = json.loads(Path('.graphify_detect.json').read_text()) -save_manifest(detect['files']) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) # Update cumulative cost tracker -extract = json.loads(Path('.graphify_extract.json').read_text()) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) cost_path = Path('graphify-out/cost.json') if cost_path.exists(): - cost = json.loads(cost_path.read_text()) + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) else: cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} @@ -678,12 +517,12 @@ cost['runs'].append({ }) cost['total_input_tokens'] += input_tok cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_chunk_*.json +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json rm -f graphify-out/.needs_update 2>/dev/null || true ``` @@ -718,521 +557,51 @@ The graph is the map. Your job after the pipeline is to be the guide. --- -## For --update (incremental re-extraction) +## Interpreter guard for subcommands -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" fi ``` +## For --update and --cluster-only -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction) - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None -new_extract = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` -Clean up after: `rm -f .graphify_old.json` - ---- - -## For --cluster-only - -Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. --- ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" +graphify query "" ``` -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - -Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. --- -## For /graphify path +## For /graphify add and --watch -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. --- -## For /graphify explain +## For the commit hook and native CLAUDE.md integration -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- - -## For /graphify add - -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text - ---- - -## For --watch - -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. - ---- - -## For git commit hook - -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - -If a post-commit hook already exists, graphify appends to it rather than replacing it. - ---- - -## For native CLAUDE.md integration - -Run once per project to make graphify always-on in Claude Code sessions: - -```bash -graphify claude install -``` - -This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. - -```bash -graphify claude uninstall # remove the section -``` +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. --- diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index f82d135d..6c4f2086 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -1,6 +1,6 @@ --- name: graphify -description: "any input (code, docs, papers, images) → knowledge graph → clustered communities → HTML + JSON + audit report. Use when user asks any question about a codebase, project content, architecture, or file relationships — especially if graphify-out/ exists. Provides persistent graph with god nodes, community detection, and BFS/DFS query tools." +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." trigger: /graphify --- @@ -13,8 +13,13 @@ Turn any folder of files into a navigable knowledge graph with community detecti ``` /graphify # full pipeline on current directory → Obsidian vault /graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON /graphify --html # (HTML is generated by default - this flag is a no-op) @@ -24,6 +29,8 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j /graphify --mcp # start MCP stdio server for agent access /graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) /graphify add # fetch URL, save to ./raw, update graph /graphify add --author "Name" # tag who wrote it /graphify add --contributor "Name" # tag who added it to the corpus @@ -36,27 +43,24 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes → one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. ## What You Must Do When Invoked If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + If no path was given, use `.` (current directory). Do not ask the user for a path. +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + Follow these steps in order. Do not skip steps. +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + ### Step 1 - Ensure graphify is installed ```bash @@ -91,8 +95,8 @@ fi # Write interpreter path for all subsequent steps (persists across invocations) mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -# Force UTF-8 I/O on Windows (prevents garbled CJK/non-ASCII output) -export PYTHONUTF8=1 +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -107,7 +111,7 @@ import json from graphify.detect import detect from pathlib import Path result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) +print(json.dumps(result, ensure_ascii=False)) " > graphify-out/.graphify_detect.json ``` @@ -127,58 +131,31 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: do not stop for an interactive subfolder choice. Show the warning, reduce semantic chunks to 10-12 files each, and continue with all supported files. Tell the user this smaller-chunk policy was applied. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. - Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. -### Step 2.5 - Transcribe video / audio files (only if video files detected) +### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. ### Step 3 - Extract entities and relationships **Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** @@ -196,16 +173,16 @@ from pathlib import Path import json code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2)) + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') " ``` @@ -218,7 +195,7 @@ else: Before dispatching subagents, print a timing estimate: - Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` by default, or `ceil(uncached_non_code_files / 11)` if the smaller-chunk large-corpus policy was applied +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) - Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) - Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" @@ -232,14 +209,14 @@ import json from graphify.cache import check_semantic_cache from pathlib import Path -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) all_files = [f for files in detect['files'].values() for f in files] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached)) + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " ``` @@ -248,7 +225,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each by default, or 10-12 files each if the smaller-chunk large-corpus policy was applied. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. **Step B2 - Dispatch ALL subagents in a single message (OpenCode)** @@ -257,97 +234,38 @@ Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-2 Dispatch one `@mention` per chunk — ALL in the same response: ``` -@agent Chunk CHUNK_NUM of TOTAL_CHUNKS: [extraction prompt below with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted] +@agent Chunk CHUNK_NUM of TOTAL_CHUNKS: [extraction prompt with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted] @agent Chunk 2 of TOTAL_CHUNKS: [next chunk] ``` -Wait for all agents to return. Parse each response as JSON. Accumulate nodes/edges/hyperedges across all results and write to `graphify-out/.graphify_semantic_new.json`. +Wait for all agents to return. Parse each response as JSON. Accumulate nodes/edges/hyperedges across all results and write to `graphify-out/.graphify_semantic_new.json`. If the `@agent` path cannot write chunk files, fall back to the serial path that writes each `graphify-out/.graphify_chunk_NN.json` before merge. -The extraction prompt each agent receives (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE): +Subagent prompt template: -``` -You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. -Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. - -Files (chunk CHUNK_NUM of TOTAL_CHUNKS): -FILE_LIST - -Rules: -- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") -- INFERRED: reasonable inference (shared data structure, implied dependency) -- AMBIGUOUS: uncertain - flag for review, do not omit - -Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). - Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant named node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use the closest existing `file_type` (`document` for prose, `code` for code-derived concepts). Do NOT invent file_types like `concept` or `rationale` — valid values are only `code|document|paper|image`. -Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. -Image files: use vision to understand what the image IS - do not just OCR. - UI screenshot: layout patterns, design decisions, key elements, purpose. - Chart: metric, trend/insight, data source. - Tweet/post: claim as node, author, concepts mentioned. - Diagram: components and connections. - Research figure: what it demonstrates, method, result. - Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. - -DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, - shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - -Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: -- Two functions that both validate user input but never call each other -- A class in code and a concept in a paper that describe the same algorithm -- Two error types that handle the same failure mode differently -Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. - -Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: -- All classes that implement a common protocol or interface -- All functions in an authentication flow (even if they don't all call each other) -- All concepts from a paper section that form one coherent idea -Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. - -If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, - contributor onto every node from that file. - -confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: -- EXTRACTED edges: confidence_score = 1.0 always -- INFERRED edges: reason about each edge individually. - Direct structural evidence (shared data structure, clear dependency): 0.8-0.9. - Reasonable inference with some uncertainty: 0.6-0.7. - Weak or speculative: 0.4-0.5. Most edges should be 0.6-0.9, not 0.5. -- AMBIGUOUS edges: 0.1-0.3 - -Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} -``` +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each agent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted. **Step B3 - Collect, cache, and merge** Wait for all subagents. For each result: - Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal - If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the OpenCode @agent dispatch did not produce a writable chunk output — print a warning: "chunk N missing from disk — OpenCode @agent dispatch did not produce a writable chunk output. Retry @agent dispatch, reduce chunk size, or use the serial fallback." Do not silently skip. +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. - If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort -If more than half the chunks failed or are missing, stop and tell the user to retry OpenCode @agent dispatch with smaller chunks or use the serial fallback, which writes `graphify-out/.graphify_chunk_NN.json` before merge. - -Serial fallback: process chunks one at a time in the main OpenCode session. For each chunk, read only that chunk's files, produce the same JSON schema, write it to `graphify-out/.graphify_chunk_NN.json`, then continue with the normal merge step below. +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash $(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path -from graphify.semantic_cleanup import load_validated_semantic_fragment, sanitize_semantic_fragment chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: - d, errors = load_validated_semantic_fragment(Path(c)) - if errors: - print(f'Skipping invalid chunk {c}: ' + '; '.join(errors[:3])) - continue - d = sanitize_semantic_fragment(d) + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) @@ -356,7 +274,7 @@ for c in chunks: Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') " ``` @@ -368,7 +286,7 @@ import json from graphify.cache import save_semantic_cache from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) print(f'Cached {saved} files') " @@ -379,10 +297,9 @@ Merge cached + new results into `graphify-out/.graphify_semantic.json`: $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path -from graphify.semantic_cleanup import sanitize_semantic_fragment -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text()) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) @@ -401,8 +318,7 @@ merged = { 'input_tokens': new.get('input_tokens', 0), 'output_tokens': new.get('output_tokens', 0), } -merged = sanitize_semantic_fragment(merged) -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` @@ -414,10 +330,9 @@ Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.gra $(cat graphify-out/.graphify_python) -c " import sys, json from pathlib import Path -from graphify.semantic_cleanup import sanitize_semantic_fragment -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text()) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text()) +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} @@ -436,8 +351,7 @@ merged = { 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } -merged = sanitize_semantic_fragment(merged) -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") total = len(merged_nodes) edges = len(merged_edges) print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') @@ -446,6 +360,8 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + ```bash mkdir -p graphify-out $(cat graphify-out/.graphify_python) -c " @@ -457,8 +373,8 @@ from graphify.report import generate from graphify.export import to_json from pathlib import Path -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = cluster(G) @@ -470,8 +386,8 @@ labels = {cid: 'Community ' + str(cid) for cid in communities} # Placeholder questions - regenerated with real labels in Step 5 questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") to_json(G, communities, 'graphify-out/graph.json') analysis = { @@ -481,7 +397,7 @@ analysis = { 'surprises': surprises, 'questions': questions, } -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") if G.number_of_nodes() == 0: print('ERROR: Graph is empty - extraction produced no nodes.') print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') @@ -509,9 +425,9 @@ from graphify.analyze import god_nodes, surprising_connections, suggest_question from graphify.report import generate from pathlib import Path -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} @@ -524,9 +440,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') " ``` @@ -540,230 +456,23 @@ Replace INPUT_PATH with the actual path. If `--obsidian` was given: +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -NODE_LIMIT = 5000 -if G.number_of_nodes() > NODE_LIMIT: - from collections import Counter - print(f'Graph has {G.number_of_nodes()} nodes (above {NODE_LIMIT} limit). Building aggregated community view...') - node_to_community = {nid: cid for cid, members in communities.items() for nid in members} - import networkx as nx_meta - meta = nx_meta.Graph() - for cid, members in communities.items(): - meta.add_node(str(cid), label=labels.get(cid, f'Community {cid}')) - edge_counts = Counter() - for u, v in G.edges(): - cu, cv = node_to_community.get(u), node_to_community.get(v) - if cu is not None and cv is not None and cu != cv: - edge_counts[(min(cu, cv), max(cu, cv))] += 1 - for (cu, cv), w in edge_counts.items(): - meta.add_edge(str(cu), str(cv), weight=w, relation=f'{w} cross-community edges', confidence='AGGREGATED') - if meta.number_of_nodes() > 1: - meta_communities = {cid: [str(cid)] for cid in communities} - member_counts = {cid: len(members) for cid, members in communities.items()} - to_html(meta, meta_communities, 'graphify-out/graph.html', community_labels=labels or None, member_counts=member_counts) - print(f'graph.html written (aggregated: {meta.number_of_nodes()} community nodes, {meta.number_of_edges()} cross-community edges)') - print('Tip: run with --obsidian for full node-level detail.') - else: - print('Single community — aggregated view not useful. Skipping graph.html.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` -### Step 6b - Wiki (only if --wiki flag) +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `graphify-out/.graphify_labels.json` is still available. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.wiki import to_wiki -from graphify.analyze import god_nodes -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} -gods = god_nodes(G) - -n = to_wiki(G, communities, 'graphify-out/wiki', community_labels=labels or None, cohesion=cohesion, god_nodes_data=gods) -print(f'Wiki: {n} articles written to graphify-out/wiki/') -print(' graphify-out/wiki/index.md -> agent entry point') -" -``` - -### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) - -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text())) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - -### Step 7b - SVG export (only if --svg flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` - -### Step 7c - GraphML export (only if --graphml flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` - -### Step 7d - MCP server (only if --mcp flag) - -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` - -### Step 8 - Token reduction benchmark (only if total_words > 5000) - -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. --- @@ -777,17 +486,19 @@ from datetime import datetime, timezone from graphify.detect import save_manifest # Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -save_manifest(detect['files']) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) cost_path = Path('graphify-out/cost.json') if cost_path.exists(): - cost = json.loads(cost_path.read_text()) + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) else: cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} @@ -799,12 +510,12 @@ cost['runs'].append({ }) cost['total_input_tokens'] += input_tok cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json rm -f graphify-out/.needs_update 2>/dev/null || true ``` @@ -839,521 +550,51 @@ The graph is the map. Your job after the pipeline is to be the guide. --- -## For --update (incremental re-extraction) +## Interpreter guard for subcommands -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" fi ``` +## For --update and --cluster-only -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction) - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text()) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` - ---- - -## For --cluster-only - -Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. --- ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" +graphify query "" ``` -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - -Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. --- -## For /graphify path +## For /graphify add and --watch -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. --- -## For /graphify explain +## For the commit hook and native CLAUDE.md integration -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- - -## For /graphify add - -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text - ---- - -## For --watch - -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. - ---- - -## For git commit hook - -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - -If a post-commit hook already exists, graphify appends to it rather than replacing it. - ---- - -## For native CLAUDE.md integration - -Run once per project to make graphify always-on in Claude Code sessions: - -```bash -graphify claude install -``` - -This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. - -```bash -graphify claude uninstall # remove the section -``` +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. --- diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index 63dfb84a..796ad8ca 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -1,6 +1,6 @@ --- name: graphify -description: "any input (code, docs, papers, images, video) → knowledge graph → clustered communities → HTML + JSON + GRAPH_REPORT.md. Use when user asks any question about a codebase, project content, architecture, or file relationships — especially if graphify-out/ exists. Provides persistent graph with god nodes, community detection, and BFS/DFS query tools." +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." --- # /graphify @@ -12,8 +12,13 @@ Turn any folder of files into a navigable knowledge graph with community detecti ``` /graphify # full pipeline on current directory → Obsidian vault /graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON /graphify --html # (HTML is generated by default - this flag is a no-op) @@ -23,6 +28,8 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j /graphify --mcp # start MCP stdio server for agent access /graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) /graphify add # fetch URL, save to ./raw, update graph /graphify add --author "Name" # tag who wrote it /graphify add --contributor "Name" # tag who added it to the corpus @@ -35,27 +42,24 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes → one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. ## What You Must Do When Invoked If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + If no path was given, use `.` (current directory). Do not ask the user for a path. +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + Follow these steps in order. Do not skip steps. +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + ### Step 1 - Ensure graphify is installed ```bash @@ -90,6 +94,8 @@ fi # Write interpreter path for all subsequent steps (persists across invocations) mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -104,8 +110,8 @@ import json from graphify.detect import detect from pathlib import Path result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) -" > .graphify_detect.json +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -124,58 +130,31 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. - Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. -### Step 2.5 - Transcribe video / audio files (only if video files detected) +### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. ### Step 3 - Extract entities and relationships **Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** @@ -193,16 +172,16 @@ from pathlib import Path import json code_files = [] -detect = json.loads(Path('.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files) - Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: - Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') " ``` @@ -211,9 +190,13 @@ else: **Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. -> **OpenClaw platform:** Multi-agent support is still early on OpenClaw. Extraction runs sequentially — you read and extract each file yourself. This is slower than parallel platforms but fully reliable. +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** -Print: `"Semantic extraction: N files (sequential — OpenClaw)"` +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" **Step B0 - Check extraction cache first** @@ -225,52 +208,59 @@ import json from graphify.cache import check_semantic_cache from pathlib import Path -detect = json.loads(Path('.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) all_files = [f for files in detect['files'].values() for f in files] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) if cached_nodes or cached_edges or cached_hyperedges: - Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " ``` -Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. -**Step B2 - Sequential extraction (OpenClaw)** +**Step B2 - Dispatch ALL subagents in a single message** -Process each file one at a time. For each file: +Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. -1. Read the file contents -2. Extract nodes, edges, and hyperedges applying the same rules: - - EXTRACTED: relationship explicit in source (import, call, citation) - - INFERRED: reasonable inference (shared structure, implied dependency) - - AMBIGUOUS: uncertain — flag it, do not omit - - Code files: semantic edges AST cannot find. Do not re-extract imports. - - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms). Do NOT invent file_types like `concept`. When adding `calls` edges: source is caller, target is callee. - - Image files: use vision — understand what the image IS, not just OCR - - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges - - Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. - - Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. - - confidence_score REQUIRED on every edge: EXTRACTED=1.0, INFERRED=0.6-0.9 (reason individually), AMBIGUOUS=0.1-0.3 -3. Accumulate results across all files +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. -Schema for each file's output: -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +Concrete example for 3 chunks: +``` +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] +``` +All three in one message. Not three separate messages. -After processing all files, write the accumulated result to `.graphify_semantic_new.json`. +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). -**Step B3 - Cache and merge** +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` -For the accumulated result: +Subagent prompt template: -If more than half the chunks failed, stop and tell the user. +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: ```bash @@ -282,7 +272,7 @@ chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: - d = json.loads(Path(c).read_text()) + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) @@ -291,7 +281,7 @@ for c in chunks: Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') " ``` @@ -303,20 +293,20 @@ import json from graphify.cache import save_semantic_cache from pathlib import Path -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) print(f'Cached {saved} files') " ``` -Merge cached + new results into `.graphify_semantic.json`: +Merge cached + new results into `graphify-out/.graphify_semantic.json`: ```bash $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path -cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) @@ -335,11 +325,11 @@ merged = { 'input_tokens': new.get('input_tokens', 0), 'output_tokens': new.get('output_tokens', 0), } -Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction @@ -348,8 +338,8 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from pathlib import Path -ast = json.loads(Path('.graphify_ast.json').read_text()) -sem = json.loads(Path('.graphify_semantic.json').read_text()) +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} @@ -368,7 +358,7 @@ merged = { 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } -Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") total = len(merged_nodes) edges = len(merged_edges) print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') @@ -377,6 +367,8 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + ```bash mkdir -p graphify-out $(cat graphify-out/.graphify_python) -c " @@ -388,8 +380,8 @@ from graphify.report import generate from graphify.export import to_json from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = cluster(G) @@ -401,8 +393,8 @@ labels = {cid: 'Community ' + str(cid) for cid in communities} # Placeholder questions - regenerated with real labels in Step 5 questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") to_json(G, communities, 'graphify-out/graph.json') analysis = { @@ -412,7 +404,7 @@ analysis = { 'surprises': surprises, 'questions': questions, } -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") if G.number_of_nodes() == 0: print('ERROR: Graph is empty - extraction produced no nodes.') print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') @@ -427,7 +419,7 @@ Replace INPUT_PATH with the actual path. ### Step 5 - Label communities -Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). Then regenerate the report and save the labels for the visualizer: @@ -440,9 +432,9 @@ from graphify.analyze import god_nodes, surprising_connections, suggest_question from graphify.report import generate from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} @@ -455,9 +447,9 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') " ``` @@ -471,178 +463,23 @@ Replace INPUT_PATH with the actual path. If `--obsidian` was given: +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -if G.number_of_nodes() > 5000: - print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` -### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('.graphify_extract.json').read_text())) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - -### Step 7b - SVG export (only if --svg flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` - -### Step 7c - GraphML export (only if --graphml flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` - -### Step 7d - MCP server (only if --mcp flag) - -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` - -### Step 8 - Token reduction benchmark (only if total_words > 5000) - -If `total_words` from `.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. --- @@ -656,17 +493,19 @@ from datetime import datetime, timezone from graphify.detect import save_manifest # Save manifest for --update -detect = json.loads(Path('.graphify_detect.json').read_text()) -save_manifest(detect['files']) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) # Update cumulative cost tracker -extract = json.loads(Path('.graphify_extract.json').read_text()) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) cost_path = Path('graphify-out/cost.json') if cost_path.exists(): - cost = json.loads(cost_path.read_text()) + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) else: cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} @@ -678,12 +517,12 @@ cost['runs'].append({ }) cost['total_input_tokens'] += input_tok cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_chunk_*.json +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json rm -f graphify-out/.needs_update 2>/dev/null || true ``` @@ -718,521 +557,51 @@ The graph is the map. Your job after the pipeline is to be the guide. --- -## For --update (incremental re-extraction) +## Interpreter guard for subcommands -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" fi ``` +## For --update and --cluster-only -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction) - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None -new_extract = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` -Clean up after: `rm -f .graphify_old.json` - ---- - -## For --cluster-only - -Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. --- ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" +graphify query "" ``` -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - -Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. --- -## For /graphify path +## For /graphify add and --watch -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. --- -## For /graphify explain +## For the commit hook and native CLAUDE.md integration -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- - -## For /graphify add - -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text - ---- - -## For --watch - -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. - ---- - -## For git commit hook - -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - -If a post-commit hook already exists, graphify appends to it rather than replacing it. - ---- - -## For native CLAUDE.md integration - -Run once per project to make graphify always-on in Claude Code sessions: - -```bash -graphify claude install -``` - -This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. - -```bash -graphify claude uninstall # remove the section -``` +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. --- diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 5481f9aa..8ba7c66f 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -1,6 +1,6 @@ --- name: graphify -description: "any input (code, docs, papers, images) → knowledge graph → clustered communities → HTML + JSON + audit report. Use when user asks any question about a codebase, project content, architecture, or file relationships — especially if graphify-out/ exists. Provides persistent graph with god nodes, community detection, and BFS/DFS query tools." +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." trigger: /graphify --- @@ -13,8 +13,13 @@ Turn any folder of files into a navigable knowledge graph with community detecti ``` /graphify # full pipeline on current directory → Obsidian vault /graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON /graphify --html # (HTML is generated by default - this flag is a no-op) @@ -24,6 +29,8 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j /graphify --mcp # start MCP stdio server for agent access /graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) /graphify add # fetch URL, save to ./raw, update graph /graphify add --author "Name" # tag who wrote it /graphify add --contributor "Name" # tag who added it to the corpus @@ -36,27 +43,24 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that an AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes → one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. ## What You Must Do When Invoked If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + If no path was given, use `.` (current directory). Do not ask the user for a path. +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + Follow these steps in order. Do not skip steps. +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + ### Step 1 - Ensure graphify is installed ```bash @@ -91,6 +95,8 @@ fi # Write interpreter path for all subsequent steps (persists across invocations) mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -105,8 +111,8 @@ import json from graphify.detect import detect from pathlib import Path result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) -" > .graphify_detect.json +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -125,52 +131,18 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. - Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. -### Step 2.5 - Transcribe video / audio files (only if video files detected) +### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. ### Step 3 - Extract entities and relationships @@ -178,6 +150,13 @@ After transcription: This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. @@ -194,16 +173,16 @@ from pathlib import Path import json code_files = [] -detect = json.loads(Path('.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files) - Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: - Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') " ``` @@ -212,10 +191,10 @@ else: **Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. -**MANDATORY: You MUST use the Agent (Task) tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `.graphify_detect.json` +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` - Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) - Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) - Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" @@ -230,91 +209,47 @@ import json from graphify.cache import check_semantic_cache from pathlib import Path -detect = json.loads(Path('.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) all_files = [f for files in detect['files'].values() for f in files] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) if cached_nodes or cached_edges or cached_hyperedges: - Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " ``` -Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. -**Step B2 - Dispatch ALL subagents using the Agent tool (Trae)** +**Step B2 - Dispatch ALL subagents in a single message** -> **Trae platform:** Uses the **Agent (Task) tool** to dispatch subagents for parallel extraction. -> Each subagent runs independently and returns structured JSON results. +> Uses the `Task` tool for parallel subagent dispatch. +> Call `Task` once per chunk — ALL in the same response so they run in parallel. > Trae does NOT support PreToolUse hooks — AGENTS.md rules are the always-on mechanism instead. -Use the **Task/Agent tool** to dispatch one subagent per chunk — launch ALL agents in parallel so they run simultaneously. Each agent receives the extraction prompt below with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted: +Pass the extraction prompt as the task description: ``` -You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. -Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. - -Files (chunk CHUNK_NUM of TOTAL_CHUNKS): -FILE_LIST - -Rules: -- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") -- INFERRED: reasonable inference (shared data structure, implied dependency) -- AMBIGUOUS: uncertain - flag for review, do not omit - -Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). - Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). Do NOT invent file_types like `concept` — valid values are only `code|document|paper|image|rationale`. -Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. -Image files: use vision to understand what the image IS - do not just OCR. - UI screenshot: layout patterns, design decisions, key elements, purpose. - Chart: metric, trend/insight, data source. - Tweet/post: claim as node, author, concepts mentioned. - Diagram: components and connections. - Research figure: what it demonstrates, method, result. - Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. - -DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, - shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - -Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: -- Two functions that both validate user input but never call each other -- A class in code and a concept in a paper that describe the same algorithm -- Two error types that handle the same failure mode differently -Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. - -Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: -- All classes that implement a common protocol or interface -- All functions in an authentication flow (even if they don't all call each other) -- All concepts from a paper section that form one coherent idea -Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. - -If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, - contributor onto every node from that file. - -confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: -- EXTRACTED edges: confidence_score = 1.0 always -- INFERRED edges: reason about each edge individually. - Direct structural evidence (shared data structure, clear dependency): 0.8-0.9. - Reasonable inference with some uncertainty: 0.6-0.7. - Weak or speculative: 0.4-0.5. Most edges should be 0.6-0.9, not 0.5. -- AMBIGUOUS edges: 0.1-0.3 - -Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") ``` -After all subagents complete, collect their results. For each result: -- If a subagent returned valid JSON with `nodes` and `edges`, include it -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. -Accumulate nodes/edges/hyperedges across all results and write to `.graphify_semantic_new.json`. +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. **Step B3 - Collect, cache, and merge** @@ -336,7 +271,7 @@ chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) all_nodes, all_edges, all_hyperedges = [], [], [] total_in, total_out = 0, 0 for c in chunks: - d = json.loads(Path(c).read_text()) + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) all_nodes += d.get('nodes', []) all_edges += d.get('edges', []) all_hyperedges += d.get('hyperedges', []) @@ -345,7 +280,7 @@ for c in chunks: Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') " ``` @@ -357,20 +292,20 @@ import json from graphify.cache import save_semantic_cache from pathlib import Path -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) print(f'Cached {saved} files') " ``` -Merge cached + new results into `.graphify_semantic.json`: +Merge cached + new results into `graphify-out/.graphify_semantic.json`: ```bash $(cat graphify-out/.graphify_python) -c " import json from pathlib import Path -cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) @@ -389,11 +324,11 @@ merged = { 'input_tokens': new.get('input_tokens', 0), 'output_tokens': new.get('output_tokens', 0), } -Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction @@ -402,9 +337,10 @@ $(cat graphify-out/.graphify_python) -c " import sys, json from pathlib import Path -ast = json.loads(Path('.graphify_ast.json').read_text()) -sem = json.loads(Path('.graphify_semantic.json').read_text()) +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) for n in sem['nodes']: @@ -421,7 +357,7 @@ merged = { 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } -Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") total = len(merged_nodes) edges = len(merged_edges) print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') @@ -430,6 +366,8 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + ```bash mkdir -p graphify-out $(cat graphify-out/.graphify_python) -c " @@ -441,8 +379,8 @@ from graphify.report import generate from graphify.export import to_json from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = cluster(G) @@ -451,10 +389,11 @@ tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get(' gods = god_nodes(G) surprises = surprising_connections(G, communities) labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") to_json(G, communities, 'graphify-out/graph.json') analysis = { @@ -464,7 +403,7 @@ analysis = { 'surprises': surprises, 'questions': questions, } -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") if G.number_of_nodes() == 0: print('ERROR: Graph is empty - extraction produced no nodes.') print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') @@ -479,7 +418,7 @@ Replace INPUT_PATH with the actual path. ### Step 5 - Label communities -Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). Then regenerate the report and save the labels for the visualizer: @@ -492,22 +431,24 @@ from graphify.analyze import god_nodes, surprising_connections, suggest_question from graphify.report import generate from pathlib import Path -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} cohesion = {int(k): v for k, v in analysis['cohesion'].items()} tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +# LABELS - replace these with the names you chose above labels = LABELS_DICT +# Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') " ``` @@ -521,166 +462,23 @@ Replace INPUT_PATH with the actual path. If `--obsidian` was given: +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -if G.number_of_nodes() > 5000: - print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` -### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('.graphify_extract.json').read_text())) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - -### Step 7b - SVG export (only if --svg flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` - -### Step 7c - GraphML export (only if --graphml flag) - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` - -### Step 7d - MCP server (only if --mcp flag) - -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. - -### Step 8 - Token reduction benchmark (only if total_words > 5000) - -If `total_words` from `.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. --- @@ -693,16 +491,20 @@ from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest -detect = json.loads(Path('.graphify_detect.json').read_text()) -save_manifest(detect['files']) +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) -extract = json.loads(Path('.graphify_extract.json').read_text()) +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) cost_path = Path('graphify-out/cost.json') if cost_path.exists(): - cost = json.loads(cost_path.read_text()) + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) else: cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} @@ -714,12 +516,12 @@ cost['runs'].append({ }) cost['total_input_tokens'] += input_tok cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_chunk_*.json +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json rm -f graphify-out/.needs_update 2>/dev/null || true ``` @@ -754,512 +556,51 @@ The graph is the map. Your job after the pipeline is to be the guide. --- -## For --update (incremental re-extraction) +## Interpreter guard for subcommands -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely, then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" fi ``` +## For --update and --cluster-only -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -new_extraction = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction) - -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None -new_extract = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` -Clean up after: `rm -f .graphify_old.json` - ---- - -## For --cluster-only - -Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. --- ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' -terms = [t.lower() for t in question.split() if len(t) > 3] - -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -token_budget = BUDGET -char_budget = token_budget * 4 - -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")}] [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" +graphify query "" ``` -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - -Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. --- -## For /graphify path +## For /graphify add and --watch -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. --- -## For /graphify explain +## For the commit hook and native AGENTS.md integration -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- - -## For /graphify add - -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- - -## For --watch - -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. - ---- - -## For git commit hook - -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - -If a post-commit hook already exists, graphify appends to it rather than replacing it. - ---- - -## For native AGENTS.md integration (Trae) - -Run once per project to make graphify always-on in Trae sessions: - -```bash -graphify trae install # or: graphify trae-cn install -``` - -This writes a `## graphify` section to the local `AGENTS.md` that instructs Trae to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. - -> **Note:** Unlike Claude Code, Trae does NOT support PreToolUse hooks. The AGENTS.md rules are the always-on mechanism — there is no automatic graph rebuild on tool use. Run `/graphify --update` manually after code changes if the graph needs refreshing. - -```bash -graphify trae uninstall # or: graphify trae-cn uninstall # remove the section -``` +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's AGENTS.md, see `references/hooks.md`. --- diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index 3f059bb1..7be667ca 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -1,6 +1,6 @@ --- name: graphify -description: "any input (code, docs, papers, images) → knowledge graph → clustered communities → HTML + JSON + audit report. Use when user asks any question about a codebase, project content, architecture, or file relationships — especially if graphify-out/ exists. Provides persistent graph with god nodes, community detection, and BFS/DFS query tools." +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." trigger: /graphify --- @@ -11,248 +11,601 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory -/graphify # full pipeline on specific path -/graphify --update # incremental - re-extract only new/changed files -/graphify --no-viz # skip visualization, just report + JSON -/graphify --wiki # build agent-crawlable wiki -/graphify query "" # BFS traversal - broad context +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node ``` +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. + ## What You Must Do When Invoked If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + If no path was given, use `.` (current directory). Do not ask the user for a path. +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + Follow these steps in order. Do not skip steps. -**All commands use `python -c "..."` syntax — no bash heredocs, no shell redirects, no `&&`/`||`. This runs correctly on Windows PowerShell and macOS/Linux alike.** +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. ### Step 1 - Ensure graphify is installed -```python -python -c "import graphify; import sys; from pathlib import Path; Path('graphify-out').mkdir(exist_ok=True); Path('graphify-out/.graphify_python').write_text(sys.executable)" +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` -If the import fails, install first: +If the import succeeds, print nothing and move straight to Step 2. -```python -python -m pip install graphifyy -q -``` - -Then re-run the Step 1 command. +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** ### Step 2 - Detect files -```python -python -c " -import json, sys +```bash +$(cat graphify-out/.graphify_python) -c " +import json from graphify.detect import detect from pathlib import Path - result = detect(Path('INPUT_PATH')) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, indent=2)) -total = result.get('total_files', 0) -words = result.get('total_words', 0) -print(f'Corpus: {total} files, ~{words} words') -for ftype, files in result.get('files', {}).items(): - if files: - print(f' {ftype}: {len(files)} files') -" +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json ``` -Replace `INPUT_PATH` with the actual path. Present a clean summary — do not dump the raw JSON. +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `total_words` > 2,000,000 OR `total_files` > 200: warn the user and ask which subfolder to run on. -- Otherwise: proceed to Step 3. +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Video and audio (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. ### Step 3 - Extract entities and relationships -#### Part A - Structural extraction (AST, free, no API cost) +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. -```python -python -c " -import json +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - p = Path(f) - code_files.extend(collect_files(p) if p.is_dir() else [p]) + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) if code_files: - result = extract(code_files) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2)) + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") print('No code files - skipping AST extraction') " ``` -#### Part B - Semantic extraction (AI, costs tokens) +#### Part B - Semantic extraction (parallel subagents) -Skip if corpus is code-only (no docs, papers, or images). +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. -Check cache first: +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** -```python -python -c " +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " import json from graphify.cache import check_semantic_cache from pathlib import Path -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) all_files = [f for files in detect['files'].values() for f in files] + cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) -if cached_nodes or cached_edges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached)) -print(f'Cache: {len(all_files)-len(uncached)} hit, {len(uncached)} need extraction') +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') " ``` -For each chunk of uncached files (20-25 files per chunk), dispatch a subagent with this prompt: +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. -``` -You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. -Output ONLY valid JSON: {"nodes": [...], "edges": [...], "hyperedges": [...]} +**Step B1 - Split into chunks** -Each node: {"id": "unique_id", "label": "Human Name", "file_type": "code|document|paper|image"} -Each edge: {"source": "id", "target": "id", "relation": "verb_phrase", "confidence": "EXTRACTED|INFERRED|AMBIGUOUS"} -hyperedges: [] unless you find a genuine group relationship +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. -Files: -FILE_LIST +**Step B2 - Dispatch subagents and paste their responses** + +> **No automated subagent tool:** this host has no parallel Agent/Task API, so +> extraction is driven by hand. Dispatch a subagent per chunk however the host +> allows (a fresh conversation, a parallel pane), then paste each response back. + +For each chunk of uncached files (20-25 per chunk), give a subagent the extraction prompt below. When it returns, paste its JSON response and write it to that chunk's file so Step B3 can collect it: + +```bash +# After pasting a subagent's JSON for chunk N, save it (replace N and PASTED_JSON): +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +cat > "${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" <<'CHUNK_JSON' +PASTED_JSON +CHUNK_JSON ``` -Collect all subagent responses and merge them: +Repeat for every chunk. Each chunk's JSON must land in its own `graphify-out/.graphify_chunk_NN.json` before Step B3 runs. -```python -python -c " -import json +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted, and write its response to that chunk's file. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob from pathlib import Path -# Merge: combine AST + cached + all semantic chunk results +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) all_nodes, all_edges, all_hyperedges = [], [], [] - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text()) -all_nodes.extend(ast.get('nodes', [])) -all_edges.extend(ast.get('edges', [])) - -cached_path = Path('graphify-out/.graphify_cached.json') -if cached_path.exists(): - cached = json.loads(cached_path.read_text()) - all_nodes.extend(cached.get('nodes', [])) - all_edges.extend(cached.get('edges', [])) - all_hyperedges.extend(cached.get('hyperedges', [])) - -# PASTE each subagent response here as chunk_1, chunk_2, etc. total_in, total_out = 0, 0 -for chunk_json in []: # replace [] with your chunk results - chunk = json.loads(chunk_json) if isinstance(chunk_json, str) else chunk_json - all_nodes.extend(chunk.get('nodes', [])) - all_edges.extend(chunk.get('edges', [])) - all_hyperedges.extend(chunk.get('hyperedges', [])) - total_in += chunk.get('input_tokens', 0) - total_out += chunk.get('output_tokens', 0) - -merged = {'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, 'input_tokens': total_in, 'output_tokens': total_out} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2)) -print(f'Merged: {len(all_nodes)} nodes, {len(all_edges)} edges') +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') " ``` -### Step 4 - Build graph and cluster - -```python -python -c " +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " import json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.analyze import god_nodes, surprising_connections +from graphify.cache import save_semantic_cache from pathlib import Path -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) + G = build_from_json(extraction) communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} gods = god_nodes(G) surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) -import networkx as nx -from networkx.readwrite import json_graph -graph_data = json_graph.node_link_data(G) -Path('graphify-out/graph.json').write_text(json.dumps(graph_data, indent=2)) -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps({ +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {}, - 'god_nodes': gods, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, 'surprises': surprises, -}, indent=2)) + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -print(f'God nodes: {[g[\"label\"] for g in gods[:5]]}') " ``` -### Step 5 - Generate report and visualization +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. -```python -python -c " -import json +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.analyze import god_nodes, surprising_connections +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions from graphify.report import generate from pathlib import Path -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -report = generate(G, communities, {}, {}, gods, surprises, extraction) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -print('GRAPH_REPORT.md written') +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') " ``` -```python -python -c " +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " import json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import to_html from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G = build_from_json(extraction) -communities = cluster(G) +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) -try: - to_html(G, communities, 'graphify-out/graph.html') - print('graph.html written') -except ValueError as e: - print(f'Visualization skipped: {e}') +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true ``` -### After completing all steps - -Print this summary: - +Tell the user (omit the obsidian line unless --obsidian was given): ``` -graphify complete - graph.json — GraphRAG-ready, queryable by MCP or CLI - graph.html — interactive visualization (open in browser) - GRAPH_REPORT.md — plain-language architecture summary +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) ``` -Read `graphify-out/GRAPH_REPORT.md` and share the **God Nodes** and **Surprising Connections** sections directly in the chat — do not ask the user to open the file themselves. +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi +``` + +## For --update and --cluster-only + +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: + +```bash +graphify query "" +``` + +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native CLAUDE.md integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 24d6800a..a15480dc 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -1,6 +1,6 @@ --- name: graphify-windows -description: "any input (code, docs, papers, images) → knowledge graph → clustered communities → HTML + JSON + audit report. Use when user asks any question about a codebase, project content, architecture, or file relationships — especially if graphify-out/ exists. Provides persistent graph with god nodes, community detection, and BFS/DFS query tools." +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." trigger: /graphify --- @@ -13,9 +13,13 @@ Turn any folder of files into a navigable knowledge graph with community detecti ``` /graphify # full pipeline on current directory → Obsidian vault /graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON /graphify --html # (HTML is generated by default - this flag is a no-op) @@ -23,10 +27,10 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --graphml # export graph.graphml (Gephi, yEd) /graphify --neo4j # generate graphify-out/cypher.txt for Neo4j /graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) /graphify --mcp # start MCP stdio server for agent access /graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) /graphify add # fetch URL, save to ./raw, update graph /graphify add --author "Name" # tag who wrote it /graphify add --contributor "Name" # tag who added it to the corpus @@ -39,27 +43,24 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes → one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. ## What You Must Do When Invoked If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + If no path was given, use `.` (current directory). Do not ask the user for a path. +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + Follow these steps in order. Do not skip steps. +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + ### Step 1 - Ensure graphify is installed ```powershell @@ -116,22 +117,24 @@ if (-not $GRAPHIFY_PYTHON) { # Save interpreter path — all subsequent steps read this $GRAPHIFY_PYTHON | Out-File -FilePath graphify-out\.graphify_python -Encoding utf8 -NoNewline +# Save scan root so `graphify update` (no args) knows where to look next time +(Resolve-Path INPUT_PATH).Path | Out-File -FilePath graphify-out\.graphify_root -Encoding utf8 -NoNewline ``` If the import succeeds, print nothing and move straight to Step 2. +**In every subsequent block, run Python through the saved interpreter — `& (Get-Content graphify-out\.graphify_python)` in place of a bare `python3` — so every step uses the interpreter that actually has graphify.** + ### Step 2 - Detect files -```powershell -@' +```bash +$(cat graphify-out/.graphify_python) -c " import json from graphify.detect import detect from pathlib import Path result = detect(Path('INPUT_PATH')) print(json.dumps(result, ensure_ascii=False)) -'@ | Out-File -FilePath graphify-out\.graphify_step_2_detect_files_3.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_2_detect_files_3.py | Out-File -FilePath graphify-out\.graphify_detect.json -Encoding utf8 -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_2_detect_files_3.py +" > graphify-out/.graphify_detect.json ``` Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: @@ -150,60 +153,31 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. - Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. -### Step 2.5 - Transcribe video / audio files (only if video files detected) +### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `$env:GRAPHIFY_WHISPER_PROMPT` before running the transcription command. - -**Step 2 - Transcribe (PowerShell):** - -```powershell -@' -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding="utf-8")) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -'@ | Out-File -FilePath graphify-out\.graphify_step_transcribe.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_transcribe.py | Out-File -FilePath graphify-out\.graphify_transcripts.json -Encoding utf8 -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_transcribe.py -``` - -After transcription: -- Read the transcript paths from `graphify-out\.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `$env:GRAPHIFY_WHISPER_MODEL = ""` before running the command above. +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. ### Step 3 - Extract entities and relationships **Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** @@ -213,38 +187,26 @@ Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is determin For any code files detected, run AST extraction in parallel with Part B subagents: -```powershell -@' -import json +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json from graphify.extract import collect_files, extract from pathlib import Path +import json +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) -def main(): - code_files = [] - detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding="utf-8")) - for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - - if code_files: - result = extract(code_files) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") - print(f'AST: {len(result["nodes"])} nodes, {len(result["edges"])} edges') - else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding="utf-8") - print('No code files - skipping AST extraction') - - -# Windows-spawn ProcessPoolExecutor (used inside extract()) re-imports this -# script in each worker; without an `if __name__ == "__main__":` guard the -# pool would recursively spawn itself. graphify v0.7.11+ falls back to -# sequential extraction if the pool dies, but the guard keeps multi-core -# extraction working on Windows. -if __name__ == '__main__': - main() -'@ | Out-File -FilePath graphify-out\.graphify_step_ast.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_ast.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_ast.py +if code_files: + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") + print('No code files - skipping AST extraction') +" ``` #### Part B - Semantic extraction (parallel subagents) @@ -263,105 +225,55 @@ Before dispatching subagents, print a timing estimate: Before dispatching any subagents, check which files already have cached extraction results: -```powershell -@' +```bash +$(cat graphify-out/.graphify_python) -c " import json from graphify.cache import check_semantic_cache from pathlib import Path -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding="utf-8")) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) all_files = [f for files in detect['files'].values() for f in files] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding="utf-8") -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding="utf-8") + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -'@ | Out-File -FilePath graphify-out\.graphify_step_3_extract_entities_and_relations_5.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_3_extract_entities_and_relations_5.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_3_extract_entities_and_relations_5.py +" ``` -Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. **Step B2 - Dispatch ALL subagents in a single message** Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. + Concrete example for 3 chunks: ``` -[Agent tool call 1: files 1-15] -[Agent tool call 2: files 16-30] -[Agent tool call 3: files 31-45] +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] ``` All three in one message. Not three separate messages. -Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE): +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```powershell +$PROJECT_ROOT = Get-Content graphify-out\.graphify_root +# Then for chunk N: $CHUNK_PATH = Join-Path $PROJECT_ROOT "graphify-out\.graphify_chunk_0N.json" ``` -You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. -Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. -Files (chunk CHUNK_NUM of TOTAL_CHUNKS): -FILE_LIST +Subagent prompt template: -Rules: -- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") -- INFERRED: reasonable inference (shared data structure, implied dependency) -- AMBIGUOUS: uncertain - flag for review, do not omit - -Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). - Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). Do NOT invent file_types like `concept` — valid values are only `code|document|paper|image|rationale`. -Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. -Image files: use vision to understand what the image IS - do not just OCR. - UI screenshot: layout patterns, design decisions, key elements, purpose. - Chart: metric, trend/insight, data source. - Tweet/post: claim as node, author, concepts mentioned. - Diagram: components and connections. - Research figure: what it demonstrates, method, result. - Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. - -DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, - shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - -Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: -- Two functions that both validate user input but never call each other -- A class in code and a concept in a paper that describe the same algorithm -- Two error types that handle the same failure mode differently -Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. - -Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: -- All classes that implement a common protocol or interface -- All functions in an authentication flow (even if they don't all call each other) -- All concepts from a paper section that form one coherent idea -Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. - -If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, - contributor onto every node from that file. - -confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: -- EXTRACTED edges: confidence_score = 1.0 always -- INFERRED edges: pick exactly ONE value from this set — never 0.5: - 0.95 direct structural evidence (shared data structure, named cross-file reference). - 0.85 strong inference (clear functional alignment, no direct symbol link). - 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). - 0.65 weak inference (thematically related, no shape evidence). - 0.55 speculative but plausible (surface-level co-occurrence only). - Models follow discrete rubrics better than continuous ranges; the bimodal - distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the - range guidance is being collapsed to a binary. If no value above fits, mark - the edge AMBIGUOUS rather than picking 0.4 or below. -- AMBIGUOUS edges: 0.1-0.3 - -Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} -``` +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. **Step B3 - Collect, cache, and merge** @@ -374,8 +286,8 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```powershell -& (Get-Content graphify-out\.graphify_python) -c " +```bash +$(cat graphify-out/.graphify_python) -c " import json, glob from pathlib import Path @@ -398,28 +310,26 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' ``` Save new results to cache: -```powershell -@' +```bash +$(cat graphify-out/.graphify_python) -c " import json from graphify.cache import save_semantic_cache from pathlib import Path -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) print(f'Cached {saved} files') -'@ | Out-File -FilePath graphify-out\.graphify_step_3_extract_entities_and_relations_6.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_3_extract_entities_and_relations_6.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_3_extract_entities_and_relations_6.py +" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```powershell -@' +```bash +$(cat graphify-out/.graphify_python) -c " import json from pathlib import Path -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} all_nodes = cached['nodes'] + new.get('nodes', []) all_edges = cached['edges'] + new.get('edges', []) @@ -438,23 +348,21 @@ merged = { 'input_tokens': new.get('input_tokens', 0), 'output_tokens': new.get('output_tokens', 0), } -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding="utf-8") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached["nodes"])} from cache, {len(new.get("nodes",[]))} new)') -'@ | Out-File -FilePath graphify-out\.graphify_step_3_extract_entities_and_relations_7.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_3_extract_entities_and_relations_7.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_3_extract_entities_and_relations_7.py +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" ``` -Clean up temp files: `Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_cached.json, graphify-out\.graphify_uncached.txt, graphify-out\.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction -```powershell -@' +```bash +$(cat graphify-out/.graphify_python) -c " import sys, json from pathlib import Path -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding="utf-8")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding="utf-8")) +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} @@ -473,20 +381,20 @@ merged = { 'input_tokens': sem.get('input_tokens', 0), 'output_tokens': sem.get('output_tokens', 0), } -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding="utf-8") +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") total = len(merged_nodes) edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast["nodes"])} AST + {len(sem["nodes"])} semantic)') -'@ | Out-File -FilePath graphify-out\.graphify_step_3_extract_entities_and_relations_8.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_3_extract_entities_and_relations_8.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_3_extract_entities_and_relations_8.py +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" ``` ### Step 4 - Build graph, cluster, analyze, generate outputs -```powershell -New-Item -ItemType Directory -Force -Path graphify-out | Out-Null -@' +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " import sys, json from graphify.build import build_from_json from graphify.cluster import cluster, score_all @@ -495,8 +403,8 @@ from graphify.report import generate from graphify.export import to_json from pathlib import Path -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding="utf-8")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding="utf-8")) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = cluster(G) @@ -508,8 +416,8 @@ labels = {cid: 'Community ' + str(cid) for cid in communities} # Placeholder questions - regenerated with real labels in Step 5 questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8") +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") to_json(G, communities, 'graphify-out/graph.json') analysis = { @@ -519,15 +427,13 @@ analysis = { 'surprises': surprises, 'questions': questions, } -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding="utf-8") +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") if G.number_of_nodes() == 0: print('ERROR: Graph is empty - extraction produced no nodes.') print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') raise SystemExit(1) print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -'@ | Out-File -FilePath graphify-out\.graphify_step_4_build_graph_cluster_analyze_ge_9.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_4_build_graph_cluster_analyze_ge_9.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_4_build_graph_cluster_analyze_ge_9.py +" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. @@ -540,8 +446,8 @@ Read `graphify-out/.graphify_analysis.json`. For each community key, look at its Then regenerate the report and save the labels for the visualizer: -```powershell -@' +```bash +$(cat graphify-out/.graphify_python) -c " import sys, json from graphify.build import build_from_json from graphify.cluster import score_all @@ -549,9 +455,9 @@ from graphify.analyze import god_nodes, surprising_connections, suggest_question from graphify.report import generate from pathlib import Path -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding="utf-8")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding="utf-8")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding="utf-8")) +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) G = build_from_json(extraction) communities = {int(k): v for k, v in analysis['communities'].items()} @@ -564,13 +470,11 @@ labels = LABELS_DICT # Regenerate questions with real community labels (labels affect question phrasing) questions = suggest_questions(G, communities, labels) -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding="utf-8") +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") print('Report updated with community labels') -'@ | Out-File -FilePath graphify-out\.graphify_step_5_label_communities_10.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_5_label_communities_10.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_5_label_communities_10.py +" ``` Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). @@ -582,220 +486,49 @@ Replace INPUT_PATH with the actual path. If `--obsidian` was given: -- If `--obsidian-dir ` was also given, use that path as the vault directory. Otherwise default to `graphify-out/obsidian`. +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. -```powershell -@' -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding="utf-8")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding="utf-8")) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -obsidian_dir = 'OBSIDIAN_DIR' # replace with --obsidian-dir value, or 'graphify-out/obsidian' if not given - -n = to_obsidian(G, communities, obsidian_dir, community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in {obsidian_dir}/') - -to_canvas(G, communities, f'{obsidian_dir}/graph.canvas', community_labels=labels or None) -print(f'Canvas: {obsidian_dir}/graph.canvas - open in Obsidian for structured community layout') -print() -print(f'Open {obsidian_dir}/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -'@ | Out-File -FilePath graphify-out\.graphify_step_6_generate_obsidian_vault_opt_in_11.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_6_generate_obsidian_vault_opt_in_11.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_6_generate_obsidian_vault_opt_in_11.py +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project ``` Generate the HTML graph (always, unless `--no-viz`): -```powershell -@' -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding="utf-8")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding="utf-8")) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -if G.number_of_nodes() > 5000: - print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -'@ | Out-File -FilePath graphify-out\.graphify_step_6_generate_obsidian_vault_opt_in_12.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_6_generate_obsidian_vault_opt_in_12.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_6_generate_obsidian_vault_opt_in_12.py +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz ``` -### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) -**If `--neo4j`** - generate a Cypher file for manual import: - -```powershell -@' -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding="utf-8"))) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -'@ | Out-File -FilePath graphify-out\.graphify_step_7_neo4j_export_only_if_neo4j_or__13.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_7_neo4j_export_only_if_neo4j_or__13.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_7_neo4j_export_only_if_neo4j_or__13.py -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```powershell -@' -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding="utf-8")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding="utf-8")) -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result["nodes"]} nodes, {result["edges"]} edges') -'@ | Out-File -FilePath graphify-out\.graphify_step_7_neo4j_export_only_if_neo4j_or__14.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_7_neo4j_export_only_if_neo4j_or__14.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_7_neo4j_export_only_if_neo4j_or__14.py -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - -### Step 7b - SVG export (only if --svg flag) - -```powershell -@' -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding="utf-8")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding="utf-8")) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -'@ | Out-File -FilePath graphify-out\.graphify_step_7b_svg_export_only_if_svg_flag_15.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_7b_svg_export_only_if_svg_flag_15.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_7b_svg_export_only_if_svg_flag_15.py -``` - -### Step 7c - GraphML export (only if --graphml flag) - -```powershell -@' -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding="utf-8")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding="utf-8")) - -G = build_from_json(extraction) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -'@ | Out-File -FilePath graphify-out\.graphify_step_7c_graphml_export_only_if_graphml_16.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_7c_graphml_export_only_if_graphml_16.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_7c_graphml_export_only_if_graphml_16.py -``` - -### Step 7d - MCP server (only if --mcp flag) - -```powershell -& (Get-Content graphify-out\.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` - -### Step 8 - Token reduction benchmark (only if total_words > 5000) - -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - -```powershell -@' -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding="utf-8")) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -'@ | Out-File -FilePath graphify-out\.graphify_step_8_token_reduction_benchmark_only_17.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_8_token_reduction_benchmark_only_17.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_8_token_reduction_benchmark_only_17.py -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. --- ### Step 9 - Save manifest, update cost tracker, clean up, and report -```powershell -@' +```bash +$(cat graphify-out/.graphify_python) -c " import json from pathlib import Path from datetime import datetime, timezone from graphify.detect import save_manifest # Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding="utf-8")) -save_manifest(detect['files']) +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding="utf-8")) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) cost_path = Path('graphify-out/cost.json') if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding="utf-8")) + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) else: cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} @@ -807,15 +540,13 @@ cost['runs'].append({ }) cost['total_input_tokens'] += input_tok cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding="utf-8") +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost["total_input_tokens"]:,} input, {cost["total_output_tokens"]:,} output ({len(cost["runs"])} runs)') -'@ | Out-File -FilePath graphify-out\.graphify_step_9_save_manifest_update_cost_trac_18.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_9_save_manifest_update_cost_trac_18.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_9_save_manifest_update_cost_trac_18.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_detect.json, graphify-out\.graphify_extract.json, graphify-out\.graphify_ast.json, graphify-out\.graphify_semantic.json, graphify-out\.graphify_analysis.json, graphify-out\.graphify_labels.json -Remove-Item -ErrorAction SilentlyContinue graphify-out/.needs_update +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true ``` Tell the user (omit the obsidian line unless --obsidian was given): @@ -849,566 +580,51 @@ The graph is the map. Your job after the pipeline is to be the guide. --- -## For --update (incremental re-extraction) +## Interpreter guard for subcommands -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: -```powershell -@' -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding="utf-8") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -'@ | Out-File -FilePath graphify-out\.graphify_step_for_update_incremental_re_extracti_19.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_for_update_incremental_re_extracti_19.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_for_update_incremental_re_extracti_19.py +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi ``` -If new files exist, first check whether all changed files are code files: +## For --update and --cluster-only -```powershell -@' -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -'@ | Out-File -FilePath graphify-out\.graphify_step_for_update_incremental_re_extracti_20.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_for_update_incremental_re_extracti_20.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_for_update_incremental_re_extracti_20.py -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```powershell -if (-not (Test-Path graphify-out\.graphify_extract.json)) { - Write-Host '[graphify update] Only deletions -- creating empty extraction for merge.' - @' -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -'@ | Out-File -FilePath graphify-out\.graphify_step_empty_extract.py -Encoding utf8 - & (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_empty_extract.py - Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_empty_extract.py -} -``` - - -Then: - -```powershell -@' -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text(encoding="utf-8")) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding="utf-8")) -G_new = build_from_json(new_extraction) - -# Prune nodes from deleted files -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding="utf-8")) -deleted = set(incremental.get('deleted_files', [])) -if deleted: - to_remove = [n for n, d in G_existing.nodes(data=True) if d.get('source_file') in deleted] - G_existing.remove_nodes_from(to_remove) - if to_remove: - print(f'Pruned {len(to_remove)} ghost node(s) from {len(deleted)} deleted file(s) — drift detected and corrected.') - else: - print(f'{len(deleted)} file(s) deleted since last run, but no ghost nodes were present in the graph — no drift.') - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') - -# Save manifest with the CURRENT full file list so the next --update -# diffs against today's filesystem state, not the prior --update's -# baseline. Without this, deleted files get reported as ghosts again -# on every subsequent --update until a full rebuild runs. -from graphify.detect import save_manifest -save_manifest(incremental['files']) -print('[graphify update] Manifest saved.') -'@ | Out-File -FilePath graphify-out\.graphify_step_for_update_incremental_re_extracti_21.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_for_update_incremental_re_extracti_21.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_for_update_incremental_re_extracti_21.py -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```powershell -@' -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding="utf-8")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding="utf-8")) -G_new = build_from_json(new_extract) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -'@ | Out-File -FilePath graphify-out\.graphify_step_for_update_incremental_re_extracti_22.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_for_update_incremental_re_extracti_22.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_for_update_incremental_re_extracti_22.py -``` - -Before the merge step, save the old graph: `Copy-Item graphify-out/graph.json graphify-out\.graphify_old.json` -Clean up after: `Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_old.json` - ---- - -## For --cluster-only - -Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```powershell -@' -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding="utf-8")) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding="utf-8") -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding="utf-8") -print(f'Re-clustered: {len(communities)} communities') -'@ | Out-File -FilePath graphify-out\.graphify_step_for_cluster_only_23.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_for_cluster_only_23.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_for_cluster_only_23.py -``` - -Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. --- ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```powershell -@' -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -'@ | Out-File -FilePath graphify-out\.graphify_step_for_graphify_query_24.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_for_graphify_query_24.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_for_graphify_query_24.py -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```powershell -@' -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding="utf-8")) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get("label",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get("label", nid)} [src={d.get("source_file","")} loc={d.get("source_location","")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get("label",u)} --{d.get("relation","")} [{d.get("confidence","")}]--> {G.nodes[v].get("label",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -'@ | Out-File -FilePath graphify-out\.graphify_step_for_graphify_query_25.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_for_graphify_query_25.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_for_graphify_query_25.py -``` - -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```powershell -& (Get-Content graphify-out\.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - -Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - ---- - -## For /graphify path - -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```powershell -@' -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -'@ | Out-File -FilePath graphify-out\.graphify_step_for_graphify_path_26.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_for_graphify_path_26.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_for_graphify_path_26.py -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```powershell -@' -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding="utf-8")) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -'@ | Out-File -FilePath graphify-out\.graphify_step_for_graphify_path_27.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_for_graphify_path_27.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_for_graphify_path_27.py -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```powershell -& (Get-Content graphify-out\.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- - -## For /graphify explain - -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```powershell -@' -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -'@ | Out-File -FilePath graphify-out\.graphify_step_for_graphify_explain_28.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_for_graphify_explain_28.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_for_graphify_explain_28.py -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```powershell -@' -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding="utf-8")) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get("label", nid)}') -print(f' source: {data_n.get("source_file","unknown")}') -print(f' type: {data_n.get("file_type","unknown")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -'@ | Out-File -FilePath graphify-out\.graphify_step_for_graphify_explain_29.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_for_graphify_explain_29.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_for_graphify_explain_29.py -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```powershell -& (Get-Content graphify-out\.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- - -## For /graphify add - -Fetch a URL and add it to the corpus, then update the graph. - -```powershell -@' -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -'@ | Out-File -FilePath graphify-out\.graphify_step_for_graphify_add_30.py -Encoding utf8 -& (Get-Content graphify-out\.graphify_python) graphify-out\.graphify_step_for_graphify_add_30.py -Remove-Item -ErrorAction SilentlyContinue graphify-out\.graphify_step_for_graphify_add_30.py -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text - ---- - -## For --watch - -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```powershell -& (Get-Content graphify-out\.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. - ---- - -## For git commit hook - -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: ```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check +graphify query "" ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - -If a post-commit hook already exists, graphify appends to it rather than replacing it. +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. --- -## For native CLAUDE.md integration +## For /graphify add and --watch -Run once per project to make graphify always-on in Claude Code sessions: +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. -```bash -graphify claude install -``` +--- -This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. +## For the commit hook and native CLAUDE.md integration -```bash -graphify claude uninstall # remove the section -``` +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. --- diff --git a/graphify/skill.md b/graphify/skill.md index 50ecd0c8..4e9827b1 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -1,6 +1,6 @@ --- name: graphify -description: "any input (code, docs, papers, images, videos) to knowledge graph. Use when user asks any question about a codebase, documents, or project content - especially if graphify-out/ exists, treat the question as a /graphify query." +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." trigger: /graphify --- @@ -57,48 +57,9 @@ If the path argument starts with `https://github.com/` or `http://github.com/`, Follow these steps in order. Do not skip steps. -### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** -```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json -``` - -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: - -```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json -``` - -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. ### Step 1 - Ensure graphify is installed @@ -179,50 +140,9 @@ Then act on it: - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. - Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. -### Step 2.5 - Transcribe video / audio files (only if video files detected) +### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `WHISPER_PROMPT` to use in the next command. - -**Step 2 - Transcribe:** - -```bash -GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths, ensure_ascii=False)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. ### Step 3 - Extract entities and relationships @@ -331,70 +251,7 @@ PROJECT_ROOT=$(cat graphify-out/.graphify_root) Subagent prompt template: -``` -You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. -Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. - -Files (chunk CHUNK_NUM of TOTAL_CHUNKS): -FILE_LIST - -Rules: -- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") -- INFERRED: reasonable inference (shared data structure, implied dependency) -- AMBIGUOUS: uncertain - flag for review, do not omit - -Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). - Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. -Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. -Image files: use vision to understand what the image IS - do not just OCR. - UI screenshot: layout patterns, design decisions, key elements, purpose. - Chart: metric, trend/insight, data source. - Tweet/post: claim as node, author, concepts mentioned. - Diagram: components and connections. - Research figure: what it demonstrates, method, result. - Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. - -DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, - shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - -Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: -- Two functions that both validate user input but never call each other -- A class in code and a concept in a paper that describe the same algorithm -- Two error types that handle the same failure mode differently -Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. - -Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: -- All classes that implement a common protocol or interface -- All functions in an authentication flow (even if they don't all call each other) -- All concepts from a paper section that form one coherent idea -Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. - -If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, - contributor onto every node from that file. - -confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: -- EXTRACTED edges: confidence_score = 1.0 always -- INFERRED edges: pick exactly ONE value from this set — never 0.5: - 0.95 direct structural evidence (shared data structure, named cross-file reference). - 0.85 strong inference (clear functional alignment, no direct symbol link). - 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). - 0.65 weak inference (thematically related, no shape evidence). - 0.55 speculative but plausible (surface-level co-occurrence only). - Models follow discrete rubrics better than continuous ranges; the bimodal - distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the - range guidance is being collapsed to a binary. If no value above fits, mark - the edge AMBIGUOUS rather than picking 0.4 or below. -- AMBIGUOUS edges: 0.1-0.3 - -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. - -Generate the extraction JSON matching this schema exactly: -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} - -Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): -CHUNK_PATH -``` +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. **Step B3 - Collect, cache, and merge** @@ -621,73 +478,9 @@ graphify export html # auto-aggregates to community view if graph > 5000 nodes # or: graphify export html --no-viz ``` -### Step 6b - Wiki (only if --wiki flag) +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - -```bash -graphify export wiki -``` - -### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) - -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -graphify export neo4j -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - -### Step 7b - SVG export (only if --svg flag) - -```bash -graphify export svg -``` - -### Step 7c - GraphML export (only if --graphml flag) - -```bash -graphify export graphml -``` - -### Step 7d - MCP server (only if --mcp flag) - -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` - -### Step 8 - Token reduction benchmark (only if total_words > 5000) - -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - -```bash -graphify benchmark -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. --- @@ -783,367 +576,33 @@ if [ ! -f graphify-out/.graphify_python ]; then fi ``` -## For --update (incremental re-extraction) +## For --update and --cluster-only -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=deleted or None, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -save_manifest(incremental['files']) -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` - ---- - -## For --cluster-only - -Skip Steps 1–3. Re-run clustering on the existing graph: - -```bash -graphify cluster-only . -``` - -Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. --- ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -### Step 0 — Constrained query expansion (REQUIRED before traversal) - -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text()) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab))) -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. - -### Step 1 — Traversal - -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, run the query directly: ```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 +graphify query "" ``` -Answer using **only** what the graph output contains. Quote `source_location` when citing a specific fact. If the graph lacks enough information, say so - do not hallucinate edges. - -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. +Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. For that vocab-expansion step, the `--dfs` / `--budget` modes, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. --- -## For /graphify path +## For /graphify add and --watch -Find the shortest path between two named concepts in the graph. - -```bash -graphify path "NODE_A" "NODE_B" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. --- -## For /graphify explain +## For the commit hook and native CLAUDE.md integration -Give a plain-language explanation of a single node - everything connected to it. - -```bash -graphify explain "NODE_NAME" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- - -## For /graphify add - -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- - -## For --watch - -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. - ---- - -## For git commit hook - -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - -If a post-commit hook already exists, graphify appends to it rather than replacing it. - ---- - -## For native CLAUDE.md integration - -Run once per project to make graphify always-on in Claude Code sessions: - -```bash -graphify claude install -``` - -This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. - -```bash -graphify claude uninstall # remove the section -``` +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. --- diff --git a/graphify/skills/amp/references/add-watch.md b/graphify/skills/amp/references/add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/graphify/skills/amp/references/add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/graphify/skills/amp/references/exports.md b/graphify/skills/amp/references/exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/graphify/skills/amp/references/exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/graphify/skills/amp/references/extraction-spec.md b/graphify/skills/amp/references/extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/graphify/skills/amp/references/extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/graphify/skills/amp/references/github-and-merge.md b/graphify/skills/amp/references/github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/graphify/skills/amp/references/github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/graphify/skills/amp/references/hooks.md b/graphify/skills/amp/references/hooks.md new file mode 100644 index 00000000..af1ac7e7 --- /dev/null +++ b/graphify/skills/amp/references/hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native AGENTS.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's AGENTS.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native AGENTS.md integration + +Run once per project to make graphify always-on in Amp sessions: + +```bash +graphify amp install +``` + +This writes a `## graphify` section to the local `AGENTS.md` that instructs Amp to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify amp uninstall # remove the section +``` diff --git a/graphify/skills/amp/references/query.md b/graphify/skills/amp/references/query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/graphify/skills/amp/references/query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/graphify/skills/amp/references/transcribe.md b/graphify/skills/amp/references/transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/graphify/skills/amp/references/transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/graphify/skills/amp/references/update.md b/graphify/skills/amp/references/update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/graphify/skills/amp/references/update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/graphify/skills/claude/references/add-watch.md b/graphify/skills/claude/references/add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/graphify/skills/claude/references/add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/graphify/skills/claude/references/exports.md b/graphify/skills/claude/references/exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/graphify/skills/claude/references/exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/graphify/skills/claude/references/extraction-spec.md b/graphify/skills/claude/references/extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/graphify/skills/claude/references/extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/graphify/skills/claude/references/github-and-merge.md b/graphify/skills/claude/references/github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/graphify/skills/claude/references/github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/graphify/skills/claude/references/hooks.md b/graphify/skills/claude/references/hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/graphify/skills/claude/references/hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/graphify/skills/claude/references/query.md b/graphify/skills/claude/references/query.md new file mode 100644 index 00000000..b08289e1 --- /dev/null +++ b/graphify/skills/claude/references/query.md @@ -0,0 +1,103 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +### Step 0 — Constrained query expansion (REQUIRED before traversal) + +graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. + +Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: + +1. Extract the token vocabulary from node labels: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, re +from pathlib import Path +data = json.loads(Path('graphify-out/graph.json').read_text()) +vocab = set() +for n in data['nodes']: + for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): + parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] + for p in parts: + t = p.lower() + if 3 <= len(t) <= 30: + vocab.add(t) +Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab))) +print(f'vocab: {len(vocab)} tokens') +" +``` + +2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: + - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. + - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. + - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. + - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. + - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. + +3. Print the selection explicitly to the user before running the query, so the expansion is auditable: +``` +Query expanded to (from graph vocab, N tokens): [token1, token2, ...] +``` +If the list is empty, say so plainly and stop — do not proceed to traversal. + +### Step 1 — Traversal + +Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) + +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +Answer using **only** what the graph output contains. Quote `source_location` when citing a specific fact. If the graph lacks enough information, say so - do not hallucinate edges. + +After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +graphify path "NODE_A" "NODE_B" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +graphify explain "NODE_NAME" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/graphify/skills/claude/references/transcribe.md b/graphify/skills/claude/references/transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/graphify/skills/claude/references/transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/graphify/skills/claude/references/update.md b/graphify/skills/claude/references/update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/graphify/skills/claude/references/update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/graphify/skills/claw/references/add-watch.md b/graphify/skills/claw/references/add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/graphify/skills/claw/references/add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/graphify/skills/claw/references/exports.md b/graphify/skills/claw/references/exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/graphify/skills/claw/references/exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/graphify/skills/claw/references/extraction-spec.md b/graphify/skills/claw/references/extraction-spec.md new file mode 100644 index 00000000..231da82b --- /dev/null +++ b/graphify/skills/claw/references/extraction-spec.md @@ -0,0 +1,29 @@ +# graphify reference: extraction subagent prompt (compact) + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation) +- INFERRED: reasonable inference (shared structure, implied dependency) +- AMBIGUOUS: uncertain — flag it, do not omit +- Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. +- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Image files: use vision — understand what the image IS, not just OCR +- DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. +- Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. +- Hyperedges: if 3+ nodes share a concept, flow, or pattern not captured by pairwise edges, add a hyperedge to a top-level `hyperedges` array. Use sparingly. Max 3 per chunk. +- If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. +- confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. + +Output exactly this JSON (no other text): +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +``` diff --git a/graphify/skills/claw/references/github-and-merge.md b/graphify/skills/claw/references/github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/graphify/skills/claw/references/github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/graphify/skills/claw/references/hooks.md b/graphify/skills/claw/references/hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/graphify/skills/claw/references/hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/graphify/skills/claw/references/query.md b/graphify/skills/claw/references/query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/graphify/skills/claw/references/query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/graphify/skills/claw/references/transcribe.md b/graphify/skills/claw/references/transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/graphify/skills/claw/references/transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/graphify/skills/claw/references/update.md b/graphify/skills/claw/references/update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/graphify/skills/claw/references/update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/graphify/skills/codex/references/add-watch.md b/graphify/skills/codex/references/add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/graphify/skills/codex/references/add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/graphify/skills/codex/references/exports.md b/graphify/skills/codex/references/exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/graphify/skills/codex/references/exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/graphify/skills/codex/references/extraction-spec.md b/graphify/skills/codex/references/extraction-spec.md new file mode 100644 index 00000000..231da82b --- /dev/null +++ b/graphify/skills/codex/references/extraction-spec.md @@ -0,0 +1,29 @@ +# graphify reference: extraction subagent prompt (compact) + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation) +- INFERRED: reasonable inference (shared structure, implied dependency) +- AMBIGUOUS: uncertain — flag it, do not omit +- Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. +- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Image files: use vision — understand what the image IS, not just OCR +- DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. +- Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. +- Hyperedges: if 3+ nodes share a concept, flow, or pattern not captured by pairwise edges, add a hyperedge to a top-level `hyperedges` array. Use sparingly. Max 3 per chunk. +- If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. +- confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. + +Output exactly this JSON (no other text): +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +``` diff --git a/graphify/skills/codex/references/github-and-merge.md b/graphify/skills/codex/references/github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/graphify/skills/codex/references/github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/graphify/skills/codex/references/hooks.md b/graphify/skills/codex/references/hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/graphify/skills/codex/references/hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/graphify/skills/codex/references/query.md b/graphify/skills/codex/references/query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/graphify/skills/codex/references/query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/graphify/skills/codex/references/transcribe.md b/graphify/skills/codex/references/transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/graphify/skills/codex/references/transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/graphify/skills/codex/references/update.md b/graphify/skills/codex/references/update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/graphify/skills/codex/references/update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/graphify/skills/copilot/references/add-watch.md b/graphify/skills/copilot/references/add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/graphify/skills/copilot/references/add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/graphify/skills/copilot/references/exports.md b/graphify/skills/copilot/references/exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/graphify/skills/copilot/references/exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/graphify/skills/copilot/references/extraction-spec.md b/graphify/skills/copilot/references/extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/graphify/skills/copilot/references/extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/graphify/skills/copilot/references/github-and-merge.md b/graphify/skills/copilot/references/github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/graphify/skills/copilot/references/github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/graphify/skills/copilot/references/hooks.md b/graphify/skills/copilot/references/hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/graphify/skills/copilot/references/hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/graphify/skills/copilot/references/query.md b/graphify/skills/copilot/references/query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/graphify/skills/copilot/references/query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/graphify/skills/copilot/references/transcribe.md b/graphify/skills/copilot/references/transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/graphify/skills/copilot/references/transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/graphify/skills/copilot/references/update.md b/graphify/skills/copilot/references/update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/graphify/skills/copilot/references/update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/graphify/skills/droid/references/add-watch.md b/graphify/skills/droid/references/add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/graphify/skills/droid/references/add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/graphify/skills/droid/references/exports.md b/graphify/skills/droid/references/exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/graphify/skills/droid/references/exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/graphify/skills/droid/references/extraction-spec.md b/graphify/skills/droid/references/extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/graphify/skills/droid/references/extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/graphify/skills/droid/references/github-and-merge.md b/graphify/skills/droid/references/github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/graphify/skills/droid/references/github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/graphify/skills/droid/references/hooks.md b/graphify/skills/droid/references/hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/graphify/skills/droid/references/hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/graphify/skills/droid/references/query.md b/graphify/skills/droid/references/query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/graphify/skills/droid/references/query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/graphify/skills/droid/references/transcribe.md b/graphify/skills/droid/references/transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/graphify/skills/droid/references/transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/graphify/skills/droid/references/update.md b/graphify/skills/droid/references/update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/graphify/skills/droid/references/update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/graphify/skills/kilo/references/add-watch.md b/graphify/skills/kilo/references/add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/graphify/skills/kilo/references/add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/graphify/skills/kilo/references/exports.md b/graphify/skills/kilo/references/exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/graphify/skills/kilo/references/exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/graphify/skills/kilo/references/extraction-spec.md b/graphify/skills/kilo/references/extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/graphify/skills/kilo/references/extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/graphify/skills/kilo/references/github-and-merge.md b/graphify/skills/kilo/references/github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/graphify/skills/kilo/references/github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/graphify/skills/kilo/references/hooks.md b/graphify/skills/kilo/references/hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/graphify/skills/kilo/references/hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/graphify/skills/kilo/references/query.md b/graphify/skills/kilo/references/query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/graphify/skills/kilo/references/query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/graphify/skills/kilo/references/transcribe.md b/graphify/skills/kilo/references/transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/graphify/skills/kilo/references/transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/graphify/skills/kilo/references/update.md b/graphify/skills/kilo/references/update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/graphify/skills/kilo/references/update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/graphify/skills/kiro/references/add-watch.md b/graphify/skills/kiro/references/add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/graphify/skills/kiro/references/add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/graphify/skills/kiro/references/exports.md b/graphify/skills/kiro/references/exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/graphify/skills/kiro/references/exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/graphify/skills/kiro/references/extraction-spec.md b/graphify/skills/kiro/references/extraction-spec.md new file mode 100644 index 00000000..231da82b --- /dev/null +++ b/graphify/skills/kiro/references/extraction-spec.md @@ -0,0 +1,29 @@ +# graphify reference: extraction subagent prompt (compact) + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation) +- INFERRED: reasonable inference (shared structure, implied dependency) +- AMBIGUOUS: uncertain — flag it, do not omit +- Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. +- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Image files: use vision — understand what the image IS, not just OCR +- DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. +- Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. +- Hyperedges: if 3+ nodes share a concept, flow, or pattern not captured by pairwise edges, add a hyperedge to a top-level `hyperedges` array. Use sparingly. Max 3 per chunk. +- If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. +- confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. + +Output exactly this JSON (no other text): +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +``` diff --git a/graphify/skills/kiro/references/github-and-merge.md b/graphify/skills/kiro/references/github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/graphify/skills/kiro/references/github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/graphify/skills/kiro/references/hooks.md b/graphify/skills/kiro/references/hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/graphify/skills/kiro/references/hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/graphify/skills/kiro/references/query.md b/graphify/skills/kiro/references/query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/graphify/skills/kiro/references/query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/graphify/skills/kiro/references/transcribe.md b/graphify/skills/kiro/references/transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/graphify/skills/kiro/references/transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/graphify/skills/kiro/references/update.md b/graphify/skills/kiro/references/update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/graphify/skills/kiro/references/update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/graphify/skills/opencode/references/add-watch.md b/graphify/skills/opencode/references/add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/graphify/skills/opencode/references/add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/graphify/skills/opencode/references/exports.md b/graphify/skills/opencode/references/exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/graphify/skills/opencode/references/exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/graphify/skills/opencode/references/extraction-spec.md b/graphify/skills/opencode/references/extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/graphify/skills/opencode/references/extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/graphify/skills/opencode/references/github-and-merge.md b/graphify/skills/opencode/references/github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/graphify/skills/opencode/references/github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/graphify/skills/opencode/references/hooks.md b/graphify/skills/opencode/references/hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/graphify/skills/opencode/references/hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/graphify/skills/opencode/references/query.md b/graphify/skills/opencode/references/query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/graphify/skills/opencode/references/query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/graphify/skills/opencode/references/transcribe.md b/graphify/skills/opencode/references/transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/graphify/skills/opencode/references/transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/graphify/skills/opencode/references/update.md b/graphify/skills/opencode/references/update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/graphify/skills/opencode/references/update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/graphify/skills/pi/references/add-watch.md b/graphify/skills/pi/references/add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/graphify/skills/pi/references/add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/graphify/skills/pi/references/exports.md b/graphify/skills/pi/references/exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/graphify/skills/pi/references/exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/graphify/skills/pi/references/extraction-spec.md b/graphify/skills/pi/references/extraction-spec.md new file mode 100644 index 00000000..231da82b --- /dev/null +++ b/graphify/skills/pi/references/extraction-spec.md @@ -0,0 +1,29 @@ +# graphify reference: extraction subagent prompt (compact) + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation) +- INFERRED: reasonable inference (shared structure, implied dependency) +- AMBIGUOUS: uncertain — flag it, do not omit +- Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. +- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Image files: use vision — understand what the image IS, not just OCR +- DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. +- Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. +- Hyperedges: if 3+ nodes share a concept, flow, or pattern not captured by pairwise edges, add a hyperedge to a top-level `hyperedges` array. Use sparingly. Max 3 per chunk. +- If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. +- confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. + +Output exactly this JSON (no other text): +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +``` diff --git a/graphify/skills/pi/references/github-and-merge.md b/graphify/skills/pi/references/github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/graphify/skills/pi/references/github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/graphify/skills/pi/references/hooks.md b/graphify/skills/pi/references/hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/graphify/skills/pi/references/hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/graphify/skills/pi/references/query.md b/graphify/skills/pi/references/query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/graphify/skills/pi/references/query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/graphify/skills/pi/references/transcribe.md b/graphify/skills/pi/references/transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/graphify/skills/pi/references/transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/graphify/skills/pi/references/update.md b/graphify/skills/pi/references/update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/graphify/skills/pi/references/update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/graphify/skills/trae/references/add-watch.md b/graphify/skills/trae/references/add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/graphify/skills/trae/references/add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/graphify/skills/trae/references/exports.md b/graphify/skills/trae/references/exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/graphify/skills/trae/references/exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/graphify/skills/trae/references/extraction-spec.md b/graphify/skills/trae/references/extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/graphify/skills/trae/references/extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/graphify/skills/trae/references/github-and-merge.md b/graphify/skills/trae/references/github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/graphify/skills/trae/references/github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/graphify/skills/trae/references/hooks.md b/graphify/skills/trae/references/hooks.md new file mode 100644 index 00000000..7c04d5b0 --- /dev/null +++ b/graphify/skills/trae/references/hooks.md @@ -0,0 +1,35 @@ +# graphify reference: commit hook and native AGENTS.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's AGENTS.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native AGENTS.md integration (Trae) + +Run once per project to make graphify always-on in Trae sessions: + +```bash +graphify trae install # or: graphify trae-cn install +``` + +This writes a `## graphify` section to the local `AGENTS.md` that instructs Trae to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +> **Note:** Unlike Claude Code, Trae does NOT support PreToolUse hooks. The AGENTS.md rules are the always-on mechanism — there is no automatic graph rebuild on tool use. Run `/graphify --update` manually after code changes if the graph needs refreshing. + +```bash +graphify trae uninstall # or: graphify trae-cn uninstall # remove the section +``` diff --git a/graphify/skills/trae/references/query.md b/graphify/skills/trae/references/query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/graphify/skills/trae/references/query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/graphify/skills/trae/references/transcribe.md b/graphify/skills/trae/references/transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/graphify/skills/trae/references/transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/graphify/skills/trae/references/update.md b/graphify/skills/trae/references/update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/graphify/skills/trae/references/update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/graphify/skills/vscode/references/add-watch.md b/graphify/skills/vscode/references/add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/graphify/skills/vscode/references/add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/graphify/skills/vscode/references/exports.md b/graphify/skills/vscode/references/exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/graphify/skills/vscode/references/exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/graphify/skills/vscode/references/extraction-spec.md b/graphify/skills/vscode/references/extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/graphify/skills/vscode/references/extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/graphify/skills/vscode/references/github-and-merge.md b/graphify/skills/vscode/references/github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/graphify/skills/vscode/references/github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/graphify/skills/vscode/references/hooks.md b/graphify/skills/vscode/references/hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/graphify/skills/vscode/references/hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/graphify/skills/vscode/references/query.md b/graphify/skills/vscode/references/query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/graphify/skills/vscode/references/query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/graphify/skills/vscode/references/transcribe.md b/graphify/skills/vscode/references/transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/graphify/skills/vscode/references/transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/graphify/skills/vscode/references/update.md b/graphify/skills/vscode/references/update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/graphify/skills/vscode/references/update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/graphify/skills/windows/references/add-watch.md b/graphify/skills/windows/references/add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/graphify/skills/windows/references/add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/graphify/skills/windows/references/exports.md b/graphify/skills/windows/references/exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/graphify/skills/windows/references/exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/graphify/skills/windows/references/extraction-spec.md b/graphify/skills/windows/references/extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/graphify/skills/windows/references/extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/graphify/skills/windows/references/github-and-merge.md b/graphify/skills/windows/references/github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/graphify/skills/windows/references/github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/graphify/skills/windows/references/hooks.md b/graphify/skills/windows/references/hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/graphify/skills/windows/references/hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/graphify/skills/windows/references/query.md b/graphify/skills/windows/references/query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/graphify/skills/windows/references/query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/graphify/skills/windows/references/transcribe.md b/graphify/skills/windows/references/transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/graphify/skills/windows/references/transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/graphify/skills/windows/references/update.md b/graphify/skills/windows/references/update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/graphify/skills/windows/references/update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/pyproject.toml b/pyproject.toml index 8c4c071d..00f6ee39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,7 +101,12 @@ packages = ["graphify"] include-package-data = false [tool.setuptools.package-data] -graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-kilo.md", "command-kilo.md", "skill-aider.md", "skill-amp.md", "skill-copilot.md", "skill-claw.md", "skill-windows.md", "skill-droid.md", "skill-trae.md", "skill-kiro.md", "skill-vscode.md", "skill-pi.md", "skill-devin.md"] +# The skill bodies ship as graphify/skill*.md (the SKILL.md a host installs is +# copied from one of these). The progressive-disclosure references sidecar ships +# under graphify/skills//references/, and the always-on injection blocks +# under graphify/always_on/. There is no graphify/skills//SKILL.md in the +# repo, so no SKILL.md glob is needed here. +graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-kilo.md", "command-kilo.md", "skill-aider.md", "skill-amp.md", "skill-copilot.md", "skill-claw.md", "skill-windows.md", "skill-droid.md", "skill-trae.md", "skill-kiro.md", "skill-vscode.md", "skill-pi.md", "skill-devin.md", "skills/*/references/*.md", "always_on/*.md"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/tests/test_install.py b/tests/test_install.py index aac533d1..773e5567 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -211,12 +211,17 @@ def test_codex_skill_contains_spawn_agent(): assert "spawn_agent" in skill -def test_codex_skill_uses_graphify_with_dirty_graph_output(): - """Codex skill must keep graph-first orientation even when graph output is dirty.""" +def test_codex_skill_uses_graphify_with_existing_graph(): + """Codex skill must keep graph-first orientation in the lean-core split. + + The progressive-disclosure split drops codex's old monolith-only "dirty + graph output" blurb; the graph-first intent now lives in the shared core's + fast-path block, which jumps straight to the query flow when a graph exists. + """ import graphify skill = (Path(graphify.__file__).parent / "skill-codex.md").read_text() - assert "Dirty `graphify-out/` artifacts are expected" in skill - assert "not a reason to skip Graphify" in skill + assert "Fast path — existing graph" in skill + assert "skip Steps 1–5 entirely and jump straight to `## For /graphify query`" in skill assert "graphify query" in skill assert "graphify explain" in skill assert "graphify path" in skill @@ -238,18 +243,25 @@ def test_opencode_skill_contains_mention(): def test_opencode_skill_uses_opencode_agent_guidance(): - """OpenCode skill must not reference Codex/Claude agent type names.""" + """OpenCode's dispatch slot uses @mention, not the Claude Agent-tool example. + + The progressive split consolidates the bespoke v8 opencode prose into the + shared core. opencode's distinguishing delta is the @mention dispatch block; + its B2 slot must carry that and must NOT carry the Claude Agent-tool example. + (The shared Step B3 re-run hint names the general-purpose agent type as the + canonical example for every host; that lives in the shared core, not in + opencode's dispatch slot.) + """ import graphify skill = (Path(graphify.__file__).parent / "skill-opencode.md").read_text() - assert "general-purpose" not in skill - assert 'subagent_type="general-purpose"' not in skill + assert "@mention" in skill assert "@agent" in skill - assert "serial fallback" in skill - assert "reduce semantic chunks to 10-12 files each" in skill - assert "10-12 files each if the smaller-chunk large-corpus policy was applied" in skill - assert "process chunks one at a time" in skill - assert "Wait for the user's answer before proceeding" not in skill + # Scope the agent-type check to opencode's dispatch slot (B2 -> B3). + b2 = skill[skill.index("**Step B2"):skill.index("**Step B3")] + assert "general-purpose" not in b2 + assert "Concrete example for 3 chunks" not in b2 + assert "OpenCode platform" in b2 def test_kilo_skill_mentions_task_tool(): @@ -269,12 +281,18 @@ def test_kilo_skill_avoids_double_quoted_python_c_fstring_dict_keys(): assert not re.search(r"print\(f'.*\[[\"'][^\"']+[\"']\]", skill) -def test_claw_skill_is_sequential(): - """OpenClaw skill file must describe sequential extraction.""" +def test_claw_skill_uses_agent_tool_dispatch(): + """OpenClaw rides the shared Agent-tool disk-collect dispatch. + + The consolidated design moves claw off the v8 sequential OpenClaw flow onto + the same agent-tool-disk dispatch as claude (per-platform-deltas), so its B2 + slot uses the Agent tool and must not carry the Codex or OpenCode mechanics. + """ import graphify skill = (Path(graphify.__file__).parent / "skill-claw.md").read_text() - assert "sequential" in skill.lower() + b2 = skill[skill.index("**Step B2"):skill.index("**Step B3")] + assert 'subagent_type="general-purpose"' in b2 assert "spawn_agent" not in skill assert "@mention" not in skill @@ -784,3 +802,104 @@ def test_gemini_uninstall_noop_if_not_installed(tmp_path): from graphify.__main__ import gemini_uninstall gemini_uninstall(tmp_path) # should not raise + + +def test_amp_user_install_lands_in_config_agents(tmp_path, monkeypatch): + """`graphify amp install` (user scope) must drop the skill into an Amp search + root: ~/.config/agents/skills, not the old ~/.amp/skills.""" + from graphify.__main__ import main + + home = tmp_path / "home" + project = tmp_path / "project" + project.mkdir() + monkeypatch.chdir(project) + monkeypatch.setattr(sys, "argv", ["graphify", "amp", "install"]) + with patch("graphify.__main__.Path.home", return_value=home): + main() + + correct = home / ".config" / "agents" / "skills" / "graphify" / "SKILL.md" + old = home / ".amp" / "skills" / "graphify" / "SKILL.md" + assert correct.exists(), f"amp skill missing from Amp search root {correct}" + assert not old.exists(), f"amp skill must not land at the unsearched {old}" + # AGENTS.md still written in the project for the always-on rules. + assert (project / "AGENTS.md").exists() + + +def test_amp_install_cleans_legacy_amp_skills_dir(tmp_path, monkeypatch): + """A pre-fix ~/.amp/skills/graphify install is removed on the next install.""" + from graphify.__main__ import main + + home = tmp_path / "home" + project = tmp_path / "project" + project.mkdir() + legacy = home / ".amp" / "skills" / "graphify" + legacy.mkdir(parents=True) + (legacy / "SKILL.md").write_text("old amp skill", encoding="utf-8") + monkeypatch.chdir(project) + monkeypatch.setattr(sys, "argv", ["graphify", "amp", "install"]) + with patch("graphify.__main__.Path.home", return_value=home): + main() + + assert not legacy.exists(), "legacy ~/.amp/skills/graphify should be cleaned up" + assert (home / ".config" / "agents" / "skills" / "graphify" / "SKILL.md").exists() + + +def test_amp_user_uninstall_removes_skill_and_agents(tmp_path, monkeypatch): + """`graphify amp uninstall` removes the user-scope skill and AGENTS.md section.""" + from graphify.__main__ import main + + home = tmp_path / "home" + project = tmp_path / "project" + project.mkdir() + monkeypatch.chdir(project) + with patch("graphify.__main__.Path.home", return_value=home): + monkeypatch.setattr(sys, "argv", ["graphify", "amp", "install"]) + main() + skill = home / ".config" / "agents" / "skills" / "graphify" / "SKILL.md" + assert skill.exists() + + monkeypatch.setattr(sys, "argv", ["graphify", "amp", "uninstall"]) + main() + + assert not skill.exists() + assert not (home / ".config" / "agents" / "skills").exists() + assert not (project / "AGENTS.md").exists() + + +def test_amp_project_install_lands_in_dot_agents(tmp_path, monkeypatch): + """Project-scope amp install lands in .agents/skills, an Amp project search root.""" + from graphify.__main__ import main + + home = tmp_path / "home" + project = tmp_path / "project" + project.mkdir() + monkeypatch.chdir(project) + monkeypatch.setattr(sys, "argv", ["graphify", "amp", "install", "--project"]) + with patch("graphify.__main__.Path.home", return_value=home): + main() + + assert (project / ".agents" / "skills" / "graphify" / "SKILL.md").exists() + assert not (project / ".amp" / "skills" / "graphify" / "SKILL.md").exists() + assert (project / "AGENTS.md").exists() + # User scope untouched. + assert not (home / ".config" / "agents" / "skills" / "graphify" / "SKILL.md").exists() + + +def test_uninstall_all_removes_amp_user_skill(tmp_path, monkeypatch): + """The user-scope `graphify uninstall` enumeration removes the amp skill.""" + from graphify.__main__ import main + + home = tmp_path / "home" + project = tmp_path / "project" + project.mkdir() + monkeypatch.chdir(project) + with patch("graphify.__main__.Path.home", return_value=home): + monkeypatch.setattr(sys, "argv", ["graphify", "amp", "install"]) + main() + skill = home / ".config" / "agents" / "skills" / "graphify" / "SKILL.md" + assert skill.exists() + + monkeypatch.setattr(sys, "argv", ["graphify", "uninstall"]) + main() + + assert not skill.exists() diff --git a/tests/test_install_references.py b/tests/test_install_references.py new file mode 100644 index 00000000..6975406f --- /dev/null +++ b/tests/test_install_references.py @@ -0,0 +1,462 @@ +"""Tests for the progressive-disclosure references/ sidecar install path. + +The real claude bundle now ships in the package (graphify/skills/claude/), so +claude and its reuse twins (antigravity, kimi) install progressively: a lean +SKILL.md plus a references/ sidecar. Every other host whose bundle has not +shipped yet still installs today's byte-identical monolith. + +The plumbing tests below stage a hand-made fake bundle in claude's slot so the +dir-copy, version-stamp, reinstall, and uninstall flow can be exercised with +fixed, asserted content. The fixture backs up the real committed bundle and +restores it on teardown, so the working tree is never disturbed. +""" +from __future__ import annotations + +import os +import shutil +import sys +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest + +import graphify +import graphify.__main__ as mainmod + + +PKG_DIR = Path(graphify.__file__).parent + + +@pytest.fixture() +def fake_bundle(): + """Stage a fake references/ bundle in claude's slot, then restore the real one. + + Yields the platform name. claude is used because its skill_refs bundle is + "claude" and it has no extra plugin wiring. The real committed bundle is + moved aside for the duration and put back afterward, so the on-disk package + is left exactly as found. + """ + platform = "claude" + bundle = mainmod._PLATFORM_CONFIG[platform]["skill_refs"] + skills_root = PKG_DIR / "skills" + bundle_dir = skills_root / bundle + refs_dir = bundle_dir / "references" + + created_root = not skills_root.exists() + backup_dir = None + if bundle_dir.exists(): + backup_dir = Path(tempfile.mkdtemp()) / "bundle_backup" + shutil.move(str(bundle_dir), str(backup_dir)) + + refs_dir.mkdir(parents=True, exist_ok=True) + (refs_dir / "extraction-spec.md").write_text("# extraction spec fragment\n", encoding="utf-8") + (refs_dir / "query.md").write_text("# query fragment\n", encoding="utf-8") + try: + yield platform + finally: + if bundle_dir.exists(): + shutil.rmtree(bundle_dir, ignore_errors=True) + if backup_dir is not None: + shutil.move(str(backup_dir), str(bundle_dir)) + shutil.rmtree(backup_dir.parent, ignore_errors=True) + elif created_root: + shutil.rmtree(skills_root, ignore_errors=True) + + +def _install(tmp_path, platform): + old_cwd = Path.cwd() + try: + os.chdir(tmp_path) + with patch("graphify.__main__.Path.home", return_value=tmp_path): + mainmod.install(platform=platform) + finally: + os.chdir(old_cwd) + + +def test_install_stages_references_sidecar(tmp_path, fake_bundle): + """A progressive platform install drops references/ alongside SKILL.md.""" + platform = fake_bundle + _install(tmp_path, platform) + skill_dir = tmp_path / ".claude" / "skills" / "graphify" + assert (skill_dir / "SKILL.md").exists() + refs = skill_dir / "references" + assert refs.is_dir() + assert (refs / "extraction-spec.md").read_text() == "# extraction spec fragment\n" + assert (refs / "query.md").read_text() == "# query fragment\n" + # No leftover staging dir. + assert not (skill_dir / "references.tmp").exists() + + +def test_single_version_stamp_covers_skill_and_references(tmp_path, fake_bundle): + """One .graphify_version stamp versions SKILL.md + references/ together.""" + platform = fake_bundle + _install(tmp_path, platform) + skill_dir = tmp_path / ".claude" / "skills" / "graphify" + stamps = list(skill_dir.rglob(".graphify_version")) + assert len(stamps) == 1 + assert stamps[0] == skill_dir / ".graphify_version" + assert stamps[0].read_text() == mainmod.__version__ + + +def test_reinstall_replaces_references_atomically(tmp_path, fake_bundle): + """Reinstall swaps references/ in place, dropping a stale fragment.""" + platform = fake_bundle + _install(tmp_path, platform) + skill_dir = tmp_path / ".claude" / "skills" / "graphify" + refs = skill_dir / "references" + # Simulate a stale fragment left from an older install. + (refs / "stale-old.md").write_text("stale\n", encoding="utf-8") + assert (refs / "stale-old.md").exists() + + _install(tmp_path, platform) + + # The stale fragment is gone; the packaged ones are present. + assert not (refs / "stale-old.md").exists() + assert (refs / "extraction-spec.md").exists() + assert (refs / "query.md").exists() + assert not (skill_dir / "references.tmp").exists() + + +def test_uninstall_removes_references_then_walks_dirs(tmp_path, fake_bundle): + """Uninstall rmtrees references/ before the dir walk so the tree is cleared.""" + platform = fake_bundle + _install(tmp_path, platform) + skill_dir = tmp_path / ".claude" / "skills" / "graphify" + assert (skill_dir / "references").is_dir() + + with patch("graphify.__main__.Path.home", return_value=tmp_path): + removed = mainmod._remove_skill_file(platform) + + assert removed + assert not skill_dir.exists() + # The 3-level walk collapsed the now-empty skill dirs. + assert not (tmp_path / ".claude" / "skills").exists() + + +def test_check_skill_version_warns_on_missing_references(tmp_path, fake_bundle, capsys): + """If SKILL.md links references/ but the dir is gone, warn to repair.""" + platform = fake_bundle + _install(tmp_path, platform) + skill_dir = tmp_path / ".claude" / "skills" / "graphify" + skill = skill_dir / "SKILL.md" + # Force the body to reference the sidecar, then delete the sidecar. + skill.write_text("See references/extraction-spec.md for the schema.\n", encoding="utf-8") + shutil.rmtree(skill_dir / "references") + + mainmod._check_skill_version(skill) + + err = capsys.readouterr().err + assert "references/ sidecar is missing" in err + + +def test_hard_fail_when_bundle_dir_present_but_references_missing(tmp_path, monkeypatch): + """A bundle dir that exists but has no references/ subdir is a malformed + package: exit 1 rather than silently shipping an empty sidecar. + + A wholly-absent bundle dir is the opposite case (a host whose wave has not + shipped) and falls back to monolith, covered by + ``test_unbuilt_bundle_host_falls_back_to_monolith``. + """ + platform = "claude" + bundle = mainmod._PLATFORM_CONFIG[platform]["skill_refs"] + skills_root = PKG_DIR / "skills" + bundle_dir = skills_root / bundle + + created_root = not skills_root.exists() + backup_dir = None + if bundle_dir.exists(): + backup_dir = Path(tempfile.mkdtemp()) / "bundle_backup" + shutil.move(str(bundle_dir), str(backup_dir)) + # Bundle dir present, but no references/ subdir inside it. + bundle_dir.mkdir(parents=True, exist_ok=True) + (bundle_dir / "SKILL.md").write_text("body\n", encoding="utf-8") + try: + with pytest.raises(SystemExit) as exc: + with patch("graphify.__main__.Path.home", return_value=tmp_path): + monkeypatch.chdir(tmp_path) + mainmod._copy_skill_file("claude") + assert exc.value.code == 1 + finally: + if bundle_dir.exists(): + shutil.rmtree(bundle_dir, ignore_errors=True) + if backup_dir is not None: + shutil.move(str(backup_dir), str(bundle_dir)) + shutil.rmtree(backup_dir.parent, ignore_errors=True) + elif created_root: + shutil.rmtree(skills_root, ignore_errors=True) + + +def _first_unbuilt_progressive_host(): + """Find a progressive host whose bundle dir has not shipped in this build. + + The wave ships bundles incrementally (claude, then codex/windows, then the + rest), so this picks whichever progressive host is still bundle-less right + now instead of hard-coding one that a later wave will build. Returns the + host name and its packaged monolith path, or (None, None) if all built. + """ + skills_root = PKG_DIR / "skills" + for name, cfg in mainmod._PLATFORM_CONFIG.items(): + bundle = cfg.get("skill_refs") + if not bundle: + continue + if (skills_root / bundle).exists(): + continue + # Resolve the packaged monolith for this host (skill.md for claude-named). + suffix = "" if name == "claude" else f"-{name}" + monolith = PKG_DIR / f"skill{suffix}.md" + if monolith.exists(): + return name, monolith + return None, None + + +def test_unbuilt_bundle_host_falls_back_to_monolith(tmp_path): + """A progressive host whose bundle has not shipped installs the monolith. + + claude/codex/windows bundles now ship; the remaining progressive hosts do + not have a bundle yet. They must still install their byte-identical monolith + with no references/ sidecar. The host is chosen dynamically so this stays + valid as later waves ship more bundles. + """ + host, monolith = _first_unbuilt_progressive_host() + if host is None: + pytest.skip("every progressive host bundle has shipped; nothing to fall back") + assert not (PKG_DIR / "skills" / mainmod._PLATFORM_CONFIG[host]["skill_refs"]).exists() + _install(tmp_path, host) + with patch("graphify.__main__.Path.home", return_value=tmp_path): + dst = mainmod._platform_skill_destination(host) + skill_dir = dst.parent + assert (skill_dir / "SKILL.md").exists() + assert not (skill_dir / "references").exists() + # Byte-identical to the packaged monolith for that host. + assert (skill_dir / "SKILL.md").read_bytes() == monolith.read_bytes() + + +def test_claude_install_ships_lean_core_and_references(tmp_path): + """claude's real bundle ships: a lean SKILL.md plus the references/ sidecar.""" + skills_claude = PKG_DIR / "skills" / "claude" / "references" + assert skills_claude.is_dir(), "claude bundle must ship in this build" + _install(tmp_path, "claude") + skill_dir = tmp_path / ".claude" / "skills" / "graphify" + skill = skill_dir / "SKILL.md" + refs = skill_dir / "references" + assert skill.exists() + assert refs.is_dir() + body = skill.read_text(encoding="utf-8") + # The installed SKILL.md is exactly the packaged lean core (skill.md IS the + # lean core now, not the old monolith). + assert body == (PKG_DIR / "skill.md").read_text(encoding="utf-8") + # The lean core points at its references and dropped the on-demand content. + assert "references/extraction-spec.md" in body + # The full embedded subagent prompt was moved out into the references. + assert '"file_type":"code|document|paper|image|rationale|concept"' not in body + # The lean core is materially smaller than the old ~1156-line monolith. + assert len(body.splitlines()) < 800 + # The version stamp covers SKILL.md + references/ together. + assert (skill_dir / ".graphify_version").read_text() == mainmod.__version__ + # The eight on-demand fragments all landed. + names = sorted(p.name for p in refs.glob("*.md")) + assert names == [ + "add-watch.md", + "exports.md", + "extraction-spec.md", + "github-and-merge.md", + "hooks.md", + "query.md", + "transcribe.md", + "update.md", + ] + + +def test_claude_twins_ride_the_claude_bundle(tmp_path): + """antigravity and kimi reuse claude's split bundle, so they go progressive too.""" + for platform in ("antigravity", "kimi"): + refs_src = mainmod._packaged_skill_refs_dir(platform) + assert refs_src is not None + assert refs_src == PKG_DIR / "skills" / "claude" / "references" + + +def test_pyproject_declares_references_globs(): + """package-data must declare the references + always-on globs that ship the bundles. + + The references sidecar ships under graphify/skills//references/ and the + always-on injection blocks under graphify/always_on/. The earlier + skills/*/SKILL.md glob matched nothing (no graphify/skills//SKILL.md + exists; the skill bodies are graphify/skill*.md, listed explicitly), so it was + removed. This test guards the real shipped layout. + """ + import tomllib + + pyproject = PKG_DIR.parent / "pyproject.toml" + if not pyproject.exists(): + pytest.skip("pyproject.toml not adjacent to package (installed wheel)") + data = tomllib.loads(pyproject.read_text(encoding="utf-8")) + pkg_data = data["tool"]["setuptools"]["package-data"]["graphify"] + assert "skills/*/references/*.md" in pkg_data + assert "always_on/*.md" in pkg_data + # The dead glob that matched no file must not creep back in. + assert "skills/*/SKILL.md" not in pkg_data + + +# The full progressive-disclosure payload the wheel must ship: 15 skill bodies, +# 104 references (13 split hosts x 8 each), and 6 always-on injection blocks. +_EXPECTED_SKILL_BODIES = ( + "skill.md", + "skill-codex.md", + "skill-opencode.md", + "skill-kilo.md", + "skill-aider.md", + "skill-amp.md", + "skill-copilot.md", + "skill-claw.md", + "skill-windows.md", + "skill-droid.md", + "skill-trae.md", + "skill-kiro.md", + "skill-vscode.md", + "skill-pi.md", + "skill-devin.md", +) +_SPLIT_HOSTS = ( + "claude", "codex", "windows", "opencode", "kilo", "copilot", + "claw", "droid", "amp", "trae", "kiro", "pi", "vscode", +) +_REFERENCE_NAMES = ( + "add-watch.md", "exports.md", "extraction-spec.md", "github-and-merge.md", + "hooks.md", "query.md", "transcribe.md", "update.md", +) +_ALWAYS_ON_NAMES = ( + "agents-md.md", "antigravity-rules.md", "claude-md.md", + "gemini-md.md", "kiro-steering.md", "vscode-instructions.md", +) + + +def _build_wheel_names(repo_root): + """Build the wheel and return the set of arcnames inside it. + + Fails loudly (not skip) when the build backend is unavailable: `build` is a + declared dev dependency, so an environment that runs this test is expected to + have it. A silent skip is how a packaging regression slips through CI. + """ + import subprocess + import sys + import tempfile + import zipfile + + try: + import build # noqa: F401 + except ImportError: + raise AssertionError( + "the 'build' module is required for the wheel-content test but is not " + "installed; it is a declared dev dependency (run `uv sync --all-extras`)" + ) + + with tempfile.TemporaryDirectory() as outdir: + result = subprocess.run( + [sys.executable, "-m", "build", "--wheel", "--no-isolation", "--outdir", outdir, str(repo_root)], + capture_output=True, + text=True, + ) + assert result.returncode == 0, ( + "wheel build failed:\n" + f"stdout:\n{result.stdout[-1000:]}\n" + f"stderr:\n{result.stderr[-1000:]}" + ) + wheels = list(Path(outdir).glob("*.whl")) + assert wheels, "no wheel was produced" + with zipfile.ZipFile(wheels[0]) as zf: + return set(zf.namelist()) + + +def test_built_wheel_ships_the_full_skill_payload(): + """The built wheel must carry every skill body, reference, and always-on block. + + This is the headline regression guard. If the package-data globs fail to match + (e.g. the stale skills/*/SKILL.md glob that matched nothing), the wheel ships a + SKILL.md with no references/ sidecar and an install silently loses every + on-demand fragment. The test asserts the whole shipped layout: 15 skill + bodies, 96 references, and 6 always-on injection blocks. It FAILS (not skips) + when the build backend is missing, because build is a declared dev dependency. + """ + repo_root = PKG_DIR.parent + if not (repo_root / "pyproject.toml").exists(): + pytest.skip("pyproject.toml not adjacent to package (installed wheel)") + # In this wave every split-host bundle ships, so its absence is a real failure, + # not a reason to skip. + assert (PKG_DIR / "skills" / "claude" / "references" / "extraction-spec.md").exists(), ( + "the claude bundle must ship in this build; the references sidecar is missing" + ) + + names = _build_wheel_names(repo_root) + + missing_bodies = [b for b in _EXPECTED_SKILL_BODIES if f"graphify/{b}" not in names] + assert not missing_bodies, f"wheel is missing skill bodies: {missing_bodies}" + assert len(_EXPECTED_SKILL_BODIES) == 15 + + missing_refs = [ + f"graphify/skills/{host}/references/{ref}" + for host in _SPLIT_HOSTS + for ref in _REFERENCE_NAMES + if f"graphify/skills/{host}/references/{ref}" not in names + ] + assert not missing_refs, f"wheel is missing references: {missing_refs}" + assert len(_SPLIT_HOSTS) * len(_REFERENCE_NAMES) == 104 + + missing_always_on = [ + f"graphify/always_on/{name}" + for name in _ALWAYS_ON_NAMES + if f"graphify/always_on/{name}" not in names + ] + assert not missing_always_on, f"wheel is missing always-on blocks: {missing_always_on}" + assert len(_ALWAYS_ON_NAMES) == 6 + + # The specific headline file that the stale glob would have dropped. + assert "graphify/skills/claude/references/extraction-spec.md" in names + assert "graphify/skills/trae/references/hooks.md" in names + # amp is now a split host too; its bundle must ship like every other. + assert "graphify/skills/amp/references/hooks.md" in names + + +def test_monolith_install_clears_orphan_references(tmp_path, fake_bundle): + """A monolith platform install removes any orphan references/ left behind.""" + # aider is a monolith (no skill_refs). Seed an orphan references/ dir at its + # destination, then install and confirm it is cleared. + skill_dst = tmp_path / ".aider" / "graphify" / "SKILL.md" + orphan = skill_dst.parent / "references" + orphan.mkdir(parents=True) + (orphan / "leftover.md").write_text("leftover\n", encoding="utf-8") + _install(tmp_path, "aider") + assert skill_dst.exists() + assert not orphan.exists() + + +def test_amp_user_install_carries_references(tmp_path, monkeypatch): + """amp is progressive: its corrected user dir also gets the references/ sidecar. + + amp's real bundle now ships in the package (graphify/skills/amp/), so the + install copies the actual committed references alongside SKILL.md. This is the + case the progressive split was built to cover: amp was the omitted 13th host. + """ + from graphify.__main__ import main + + assert (PKG_DIR / "skills" / "amp" / "references" / "hooks.md").exists(), ( + "amp's references bundle must ship in this build" + ) + + home = tmp_path / "home" + project = tmp_path / "project" + project.mkdir() + monkeypatch.chdir(project) + with patch("graphify.__main__.Path.home", return_value=home): + monkeypatch.setattr(sys, "argv", ["graphify", "amp", "install"]) + main() + skill_dir = home / ".config" / "agents" / "skills" / "graphify" + assert (skill_dir / "SKILL.md").exists() + # A representative reference from the shipped amp bundle lands too. + assert (skill_dir / "references" / "exports.md").exists() + assert (skill_dir / "references" / "hooks.md").exists() + + monkeypatch.setattr(sys, "argv", ["graphify", "amp", "uninstall"]) + main() + + assert not skill_dir.exists() diff --git a/tests/test_install_roundtrip.py b/tests/test_install_roundtrip.py new file mode 100644 index 00000000..68216e6b --- /dev/null +++ b/tests/test_install_roundtrip.py @@ -0,0 +1,318 @@ +"""Full per-platform install + uninstall round-trip suite. + +Every platform graphify knows about installs its SKILL.md at a specific real +destination (the ``_platform_skill_destination`` map), optionally with a +references/ sidecar, and uninstall must put the tree back exactly as it found +it. This file walks every platform through that round trip at both project and +user scope, then covers the two flows that are easy to break in the +progressive-disclosure work: a monolith -> progressive upgrade, and crash +recovery when a references staging step is interrupted. + +The skill source and references bundles are the real committed package files, so +these tests exercise the actual on-disk layout an end user gets. Only Path.home +and the cwd are redirected into tmp_path so nothing touches the developer's home +directory. +""" +from __future__ import annotations + +import os +import shutil +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest + +import graphify +import graphify.__main__ as mainmod + + +PKG_DIR = Path(graphify.__file__).parent + +# Every platform in the config plus the scope each is exercised at. The +# destination is resolved from _platform_skill_destination so the assertions +# track the real install map (including amp's corrected .agents path). +ALL_CONFIG_PLATFORMS = sorted(mainmod._PLATFORM_CONFIG) + + +def _has_real_bundle(platform: str) -> bool: + """True if this platform's references bundle ships in the package right now.""" + bundle = mainmod._PLATFORM_CONFIG[platform].get("skill_refs") + if not bundle: + return False + return (PKG_DIR / "skills" / bundle / "references").is_dir() + + +@pytest.mark.parametrize("platform", ALL_CONFIG_PLATFORMS) +@pytest.mark.parametrize("project", [False, True], ids=["user", "project"]) +def test_skill_roundtrip_at_real_destination(platform, project, tmp_path, monkeypatch): + """Install then uninstall every platform's SKILL.md at its real destination. + + Install must land the skill exactly where _platform_skill_destination says, + stamp the version, and stage references/ iff the bundle ships. Uninstall must + remove the skill, the stamp, the references, and walk the now-empty dirs away. + """ + home = tmp_path / "home" + project_dir = tmp_path / "proj" + home.mkdir() + project_dir.mkdir() + monkeypatch.chdir(project_dir) + + with patch("graphify.__main__.Path.home", return_value=home): + dst = mainmod._platform_skill_destination( + platform, project=project, project_dir=project_dir + ) + # Sanity: a user-scope install must not write under the project dir, and + # vice versa, so the two scopes never collide in this test. + if project: + assert str(dst).startswith(str(project_dir)) + else: + assert str(dst).startswith(str(home)) + + returned = mainmod._copy_skill_file( + platform, project=project, project_dir=project_dir + ) + assert returned == dst + assert dst.exists(), f"{platform} ({'project' if project else 'user'}) skill not installed" + assert (dst.parent / ".graphify_version").read_text() == mainmod.__version__ + + refs = dst.parent / "references" + if _has_real_bundle(platform): + assert refs.is_dir(), f"{platform} ships a bundle but no references/ installed" + assert (refs / "extraction-spec.md").exists() + else: + assert not refs.exists(), f"{platform} is monolith but references/ appeared" + # No staging dir is ever left behind. + assert not (dst.parent / "references.tmp").exists() + + removed = mainmod._remove_skill_file( + platform, project=project, project_dir=project_dir + ) + assert removed + assert not dst.exists() + assert not (dst.parent / ".graphify_version").exists() + assert not refs.exists() + + +def test_amp_user_install_at_corrected_agents_path(tmp_path, monkeypatch): + """amp's user-scope skill lands under ~/.config/agents/skills (the fix), not ~/.amp.""" + home = tmp_path / "home" + home.mkdir() + monkeypatch.chdir(tmp_path) + with patch("graphify.__main__.Path.home", return_value=home): + dst = mainmod._copy_skill_file("amp", project=False) + assert dst == home / ".config" / "agents" / "skills" / "graphify" / "SKILL.md" + assert dst.exists() + # The legacy ~/.amp/skills location is not written. + assert not (home / ".amp" / "skills").exists() + mainmod._remove_skill_file("amp", project=False) + assert not dst.exists() + + +def test_amp_project_install_at_agents_path(tmp_path, monkeypatch): + """amp's project-scope skill lands under .agents/skills, an Amp search root.""" + project_dir = tmp_path / "proj" + project_dir.mkdir() + monkeypatch.chdir(project_dir) + with patch("graphify.__main__.Path.home", return_value=tmp_path / "home"): + dst = mainmod._copy_skill_file("amp", project=True, project_dir=project_dir) + assert dst == project_dir / ".agents" / "skills" / "graphify" / "SKILL.md" + assert dst.exists() + mainmod._remove_skill_file("amp", project=True, project_dir=project_dir) + assert not dst.exists() + + +def test_vscode_install_uninstall_roundtrip(tmp_path, monkeypatch): + """VS Code Copilot Chat round trip at ~/.copilot/skills/graphify + instructions file.""" + home = tmp_path / "home" + project_dir = tmp_path / "proj" + home.mkdir() + project_dir.mkdir() + monkeypatch.chdir(project_dir) + with patch("graphify.__main__.Path.home", return_value=home): + mainmod.vscode_install(project_dir=project_dir) + skill = home / ".copilot" / "skills" / "graphify" / "SKILL.md" + instructions = project_dir / ".github" / "copilot-instructions.md" + assert skill.exists() + assert instructions.exists() + assert "## graphify" in instructions.read_text(encoding="utf-8") + assert (skill.parent / ".graphify_version").read_text() == mainmod.__version__ + + mainmod.vscode_uninstall(project_dir=project_dir) + assert not skill.exists() + # The skill dir tree is walked away. + assert not (home / ".copilot" / "skills").exists() + # The graphify section is stripped from the instructions file. + if instructions.exists(): + assert "## graphify" not in instructions.read_text(encoding="utf-8") + + +def _install_via_entrypoint(tmp_path, platform): + """Drive the high-level install() entry point with home + cwd in tmp_path.""" + old_cwd = Path.cwd() + try: + os.chdir(tmp_path) + with patch("graphify.__main__.Path.home", return_value=tmp_path): + mainmod.install(platform=platform) + finally: + os.chdir(old_cwd) + + +def _copy_in_tmp(tmp_path, platform): + """Run _copy_skill_file with home + cwd redirected into tmp_path, restoring cwd.""" + old_cwd = Path.cwd() + try: + os.chdir(tmp_path) + with patch("graphify.__main__.Path.home", return_value=tmp_path): + mainmod._copy_skill_file(platform) + finally: + os.chdir(old_cwd) + + +def test_install_entrypoint_roundtrip_for_progressive_and_monolith(tmp_path): + """The public install() entry point round-trips a progressive and a monolith host. + + claude ships a real references bundle (progressive); aider is a monolith. + Both install through install() and uninstall through _remove_skill_file, + landing at and clearing their real destinations. + """ + for platform, rel in ( + ("claude", Path(".claude") / "skills" / "graphify"), + ("aider", Path(".aider") / "graphify"), + ): + _install_via_entrypoint(tmp_path, platform) + skill_dir = tmp_path / rel + assert (skill_dir / "SKILL.md").exists() + if _has_real_bundle(platform): + assert (skill_dir / "references").is_dir() + else: + assert not (skill_dir / "references").exists() + with patch("graphify.__main__.Path.home", return_value=tmp_path): + mainmod._remove_skill_file(platform) + assert not (skill_dir / "SKILL.md").exists() + + +# --- monolith -> progressive upgrade path -------------------------------------- + + +@pytest.fixture() +def fake_progressive_bundle(): + """Stage a controllable references bundle in claude's slot. + + Lets a test flip a host between "monolith on disk" and "progressive on disk" + deterministically without depending on the real fragment content. The real + committed bundle is moved aside and restored on teardown. + """ + platform = "claude" + bundle = mainmod._PLATFORM_CONFIG[platform]["skill_refs"] + skills_root = PKG_DIR / "skills" + bundle_dir = skills_root / bundle + refs_dir = bundle_dir / "references" + + created_root = not skills_root.exists() + backup_dir = None + if bundle_dir.exists(): + backup_dir = Path(tempfile.mkdtemp()) / "bundle_backup" + shutil.move(str(bundle_dir), str(backup_dir)) + + refs_dir.mkdir(parents=True, exist_ok=True) + (refs_dir / "extraction-spec.md").write_text("# spec\n", encoding="utf-8") + (refs_dir / "query.md").write_text("# query\n", encoding="utf-8") + try: + yield platform, bundle_dir, refs_dir + finally: + if bundle_dir.exists(): + shutil.rmtree(bundle_dir, ignore_errors=True) + if backup_dir is not None: + shutil.move(str(backup_dir), str(bundle_dir)) + shutil.rmtree(backup_dir.parent, ignore_errors=True) + elif created_root: + shutil.rmtree(skills_root, ignore_errors=True) + + +def test_monolith_to_progressive_upgrade(tmp_path, fake_progressive_bundle): + """A pre-progressive install (SKILL.md, no references/) gains references/ on upgrade.""" + platform, bundle_dir, _ = fake_progressive_bundle + skill_dir = tmp_path / ".claude" / "skills" / "graphify" + + # Simulate the old on-disk state: a monolithic SKILL.md with no sidecar. + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text("old monolith body\n", encoding="utf-8") + (skill_dir / ".graphify_version").write_text("0.0.1", encoding="utf-8") + assert not (skill_dir / "references").exists() + + # Upgrade: the platform now ships a bundle, so install adds references/. + _copy_in_tmp(tmp_path, platform) + + refs = skill_dir / "references" + assert refs.is_dir() + assert (refs / "extraction-spec.md").read_text() == "# spec\n" + assert (skill_dir / ".graphify_version").read_text() == mainmod.__version__ + assert not (skill_dir / "references.tmp").exists() + + +def test_progressive_to_monolith_downgrade_clears_references(tmp_path, fake_progressive_bundle): + """If a host loses its bundle, the next install clears the orphan references/.""" + platform, bundle_dir, _ = fake_progressive_bundle + skill_dir = tmp_path / ".claude" / "skills" / "graphify" + + # First install: progressive, references/ present. + _copy_in_tmp(tmp_path, platform) + assert (skill_dir / "references").is_dir() + + # The bundle disappears (downgrade / build without this wave). + shutil.rmtree(bundle_dir) + _copy_in_tmp(tmp_path, platform) + + assert (skill_dir / "SKILL.md").exists() + assert not (skill_dir / "references").exists(), "orphan references/ was not cleared" + + +# --- crash safety -------------------------------------------------------------- + + +def test_interrupted_references_staging_self_heals(tmp_path, fake_progressive_bundle): + """A leftover references.tmp from a crashed install is cleared on the next install.""" + platform, _, _ = fake_progressive_bundle + skill_dir = tmp_path / ".claude" / "skills" / "graphify" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text("body\n", encoding="utf-8") + + # Simulate a crash mid-stage: a half-written references.tmp is on disk. + staged = skill_dir / "references.tmp" + staged.mkdir() + (staged / "garbage.md").write_text("partial\n", encoding="utf-8") + + _copy_in_tmp(tmp_path, platform) + + refs = skill_dir / "references" + assert refs.is_dir() + assert (refs / "extraction-spec.md").exists() + # The stale staging dir and its garbage are gone. + assert not staged.exists() + assert not (refs / "garbage.md").exists() + + +def test_failed_copytree_leaves_no_partial_references(tmp_path, fake_progressive_bundle): + """If copytree blows up mid-stage, no half-written references/ is left visible.""" + platform, _, _ = fake_progressive_bundle + skill_dir = tmp_path / ".claude" / "skills" / "graphify" + skill_dir.mkdir(parents=True) + skill_dst = skill_dir / "SKILL.md" + skill_dst.write_text("body\n", encoding="utf-8") + + # A pre-existing good references/ that must survive a failed upgrade attempt. + good = skill_dir / "references" + good.mkdir() + (good / "keep.md").write_text("keep\n", encoding="utf-8") + + boom = RuntimeError("disk full") + with patch("graphify.__main__.shutil.copytree", side_effect=boom): + with pytest.raises(RuntimeError): + mainmod._install_skill_references(skill_dst, PKG_DIR / "skills" / "claude" / "references") + + # The staging dir is cleaned up and the existing references/ is untouched + # (the swap only happens after a successful copytree). + assert not (skill_dir / "references.tmp").exists() + assert good.is_dir() + assert (good / "keep.md").read_text() == "keep\n" diff --git a/tests/test_skillgen.py b/tests/test_skillgen.py new file mode 100644 index 00000000..fcac56db --- /dev/null +++ b/tests/test_skillgen.py @@ -0,0 +1,788 @@ +"""Tests for the tools/skillgen generator and the claude lean-core split. + +skillgen renders graphify's committed skill artifacts from human-edited +fragments. These tests lock in the anti-drift guards (``--check``, +``--audit-coverage``), the render idempotency, and the lean-core invariant: the +core runs a default extraction with zero reference reads, on-demand content +lives only in the references, and no reference duplicates core content. +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +# tests/ -> repo root is one parent up; put it on the path so tools.skillgen +# imports regardless of pytest's import mode. +REPO_ROOT = Path(__file__).resolve().parent.parent +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from tools.skillgen import gen # noqa: E402 + + +def test_audit_coverage_passes(): + """Every v8 heading lands in the lean core or exactly one reference.""" + platforms = gen.load_platforms() + problems = gen.audit_coverage(platforms["claude"]) + assert problems == [], "\n".join(problems) + + +def test_check_passes(): + """The committed artifacts and the expected/ snapshot match a fresh render. + + This is the CI / pre-commit drift guard. A failure here means someone + hand-edited a generated file or forgot to re-run the generator. + """ + platforms = gen.load_platforms() + artifacts = gen.render_all(platforms, only="claude") + problems = gen.check(artifacts) + assert problems == [], "\n".join(problems) + + +def test_render_is_idempotent(): + """Rendering twice yields byte-identical output (no timestamps/versions).""" + platforms = gen.load_platforms() + first = gen.render_all(platforms, only="claude") + second = gen.render_all(platforms, only="claude") + assert [(a.path, a.content) for a in first] == [(a.path, a.content) for a in second] + + +def test_render_output_is_lf_only(): + """Generated artifacts use LF newlines and end in exactly one newline.""" + platforms = gen.load_platforms() + for art in gen.render_all(platforms, only="claude"): + assert "\r" not in art.content, art.path + assert art.content.endswith("\n"), art.path + assert not art.content.endswith("\n\n"), art.path + + +def test_no_version_or_timestamp_in_output(): + """No generated artifact carries the package version string.""" + from graphify.__main__ import __version__ + + platforms = gen.load_platforms() + for art in gen.render_all(platforms, only="claude"): + assert __version__ not in art.content, f"{art.path} leaked a version string" + + +def _claude_artifacts(): + platforms = gen.load_platforms() + arts = gen.render_all(platforms, only="claude") + core = next(a for a in arts if a.path == "graphify/skill.md") + refs = {a.path.rsplit("/", 1)[-1]: a.content for a in arts if a.path != "graphify/skill.md"} + return core.content, refs + + +def test_lean_core_has_no_reference_only_content(): + """The core must not inline the execution detail of an on-demand reference. + + The ``## Usage`` flag table in the core deliberately lists every command, + including the on-demand ones (it is the --help payload), so the markers + below are execution-detail lines that never appear in that table. + """ + core, _ = _claude_artifacts() + # The full embedded subagent prompt lives only in extraction-spec.md. + assert '"file_type":"code|document|paper|image|rationale|concept"' not in core + # The incremental-update merge machinery lives only in update.md. + assert "from graphify.build import build_merge" not in core + assert "graphify cluster-only ." not in core + # The vocab-expansion query flow lives only in query.md. + assert "Constrained query expansion" not in core + assert "save-result --question" not in core + # The export commands live only in exports.md. + assert "graphify export wiki" not in core + assert "graphify export neo4j" not in core + # The add / watch / hook flows live only in their references. + assert "from graphify.ingest import ingest" not in core + assert "graphify hook install" not in core + assert "python3 -m graphify.watch" not in core + + +def test_lean_core_runs_default_pipeline_with_zero_references(): + """The default code-corpus run must be fully described inside the core.""" + core, _ = _claude_artifacts() + # The whole default pipeline (detect -> AST -> build -> label -> HTML -> + # report) must be present in the core so a plain run reads no reference. + for needed in ( + "### Step 1 - Ensure graphify is installed", + "### Step 2 - Detect files", + "### Step 3 - Extract entities and relationships", + "#### Part A - Structural extraction for code files", + "#### Part C - Merge AST + semantic into final extraction", + "### Step 4 - Build graph, cluster, analyze, generate outputs", + "### Step 5 - Label communities", + "### Step 6 - Generate Obsidian vault (opt-in) + HTML", + "### Step 9 - Save manifest, update cost tracker, clean up, and report", + "## Honesty Rules", + "graphify export html", + ): + assert needed in core, f"lean core is missing default-pipeline content: {needed!r}" + + +def test_references_contain_no_core_pipeline_content(): + """No reference fragment may duplicate the core build pipeline.""" + _, refs = _claude_artifacts() + # Distinctive lines from the core build/label steps must not appear in any + # reference, or the same content would be double-homed. + core_only_markers = ( + "from graphify.cluster import cluster, score_all", + "### Step 4 - Build graph, cluster, analyze, generate outputs", + "### Step 5 - Label communities", + "## Honesty Rules", + ) + for name, body in refs.items(): + for marker in core_only_markers: + assert marker not in body, f"reference {name} leaked core content: {marker!r}" + + +def test_reference_pointers_in_core_resolve_to_real_fragments(): + """Every references/.md the core points at is actually rendered.""" + import re + + core, refs = _claude_artifacts() + pointed = set(re.findall(r"references/([\w-]+)\.md", core)) + rendered = {name[: -len(".md")] for name in refs} + missing = pointed - rendered + assert not missing, f"core points at references that were not rendered: {missing}" + + +def test_query_heading_is_homed_in_core_stub_only(): + """The query section heading is the lean-core stub; query.md re-homes the rest.""" + core, refs = _claude_artifacts() + core_headings = set(gen.headings(core)) + query_headings = set(gen.headings(refs["query.md"])) + assert "## For /graphify query" in core_headings + assert "## For /graphify query" not in query_headings + # The deeper query content moved into the reference. + assert "## For /graphify path" in query_headings + assert "## For /graphify explain" in query_headings + assert "## For /graphify path" not in core_headings + + +def test_eight_references_render_for_claude(): + """claude renders exactly the eight on-demand fragments from the design.""" + _, refs = _claude_artifacts() + assert sorted(refs) == [ + "add-watch.md", + "exports.md", + "extraction-spec.md", + "github-and-merge.md", + "hooks.md", + "query.md", + "transcribe.md", + "update.md", + ] + + +def test_headings_helper_ignores_code_fence_comments(): + """The fence-aware heading scanner must skip '#' lines inside code fences.""" + md = ( + "# Real Heading\n" + "\n" + "```bash\n" + "# not a heading, a shell comment\n" + "echo hi\n" + "```\n" + "\n" + "## Another Real One\n" + ) + assert gen.headings(md) == ["# Real Heading", "## Another Real One"] + + +def test_enum_is_full_six_value_superset_in_extraction_spec(): + """Decision A: the file_type enum is the full six-value superset.""" + _, refs = _claude_artifacts() + spec = refs["extraction-spec.md"] + assert "`code`, `document`, `paper`, `image`, `rationale`, `concept`" in spec + assert '"file_type":"code|document|paper|image|rationale|concept"' in spec + + +# --- codex + windows (the divergent split hosts) ------------------------------- + + +def _platform_artifacts(key): + platforms = gen.load_platforms() + arts = gen.render_all(platforms, only=key) + skill_dst = platforms[key].skill_dst + core = next(a for a in arts if a.path == skill_dst) + refs = {a.path.rsplit("/", 1)[-1]: a.content for a in arts if a.path != skill_dst} + return core.content, refs + + +def test_check_passes_for_codex_and_windows(): + """The committed codex/windows artifacts match a fresh render and expected/.""" + platforms = gen.load_platforms() + for key in ("codex", "windows"): + artifacts = gen.render_all(platforms, only=key) + problems = gen.check(artifacts) + assert problems == [], f"[{key}]\n" + "\n".join(problems) + + +def test_audit_coverage_passes_for_codex_and_windows(): + """Every v8 heading single-homes for the cli-inline split hosts too.""" + platforms = gen.load_platforms() + for key in ("codex", "windows"): + problems = gen.audit_coverage(platforms[key]) + assert problems == [], f"[{key}]\n" + "\n".join(problems) + + +UNIFIED_DESCRIPTION = ( + "Use for any question about a codebase, its architecture, file relationships, " + "or project content — especially when graphify-out/ exists, where the question " + "should be treated as a graphify query first. Turns any input (code, docs, " + "papers, images, videos) into a persistent knowledge graph with god nodes, " + "community detection, and query/path/explain tools." +) + + +def test_descriptions_are_unified(): + """Every platform now carries one unified frontmatter description, byte for byte. + + The two drifted v8 descriptions (claude's short one and the richer 14-host + line) were collapsed into a single discovery-tuned line that leads with the + use-condition. Every split host and both monoliths must carry it verbatim, + and none of the old wording may survive. + """ + expected_line = f'description: "{UNIFIED_DESCRIPTION}"' + platforms = gen.load_platforms() + for key, p in platforms.items(): + body = gen.render(p)[0].content + assert expected_line in body, f"[{key}] missing the unified description line" + # None of the drifted v8 wording may survive on any platform. + assert "Provides persistent graph with god nodes" not in body, f"[{key}] kept old wording" + assert "treat the question as a /graphify query." not in body, f"[{key}] kept old wording" + assert "clustered communities" not in body, f"[{key}] kept old wording" + + +def test_windows_frontmatter_name_and_shell_and_extra(): + """windows: graphify-windows name, powershell install, troubleshooting tail.""" + core, _ = _platform_artifacts("windows") + assert core.startswith("---\nname: graphify-windows\n") + assert "```powershell" in core + assert "function Find-GraphifyPython" in core + assert "## Troubleshooting" in core + assert "### PowerShell 5.1: Vertical scrolling stops working" in core + # The troubleshooting section sits before Honesty Rules, single separator. + assert "\n4. **Skip graspologic**" in core + assert core.index("## Troubleshooting") < core.index("## Honesty Rules") + + +def test_codex_dispatch_is_agenttask_and_collects_in_memory(): + """codex: spawn/wait/close_agent dispatch needing multi_agent = true.""" + core, _ = _platform_artifacts("codex") + assert "spawn_agent" in core + assert "wait_agent" in core + assert "close_agent" in core + assert "multi_agent = true" in core + assert "Codex collects in memory" in core + # The B2 dispatch slot itself (Codex heading -> Step B3) must not carry the + # claude Agent-tool example. The shared Step B3 prose mentions the agent type + # in a re-run hint, so scope the check to the dispatch block only. + b2 = core[core.index("**Step B2"):core.index("**Step B3")] + assert "Concrete example for 3 chunks" not in b2 + assert "Agent tool call 1" not in b2 + + +def test_codex_and_windows_unify_enum_to_six_values(): + """codex (was 4-value) and windows (was 5-value) now carry the superset.""" + for key in ("codex", "windows"): + _, refs = _platform_artifacts(key) + spec = refs["extraction-spec.md"] + assert "`code`, `document`, `paper`, `image`, `rationale`, `concept`" in spec + assert '"file_type":"code|document|paper|image|rationale|concept"' in spec + # No legacy 4-value enum survives anywhere in the rendered bundle. + for body in refs.values(): + assert '"file_type":"code|document|paper|image"' not in body + + +def test_codex_uses_compact_extraction_windows_uses_verbose(): + """The extraction variant differs: codex compact, windows verbose.""" + _, codex_refs = _platform_artifacts("codex") + _, windows_refs = _platform_artifacts("windows") + assert "(compact)" in codex_refs["extraction-spec.md"] + assert "(compact)" not in windows_refs["extraction-spec.md"] + + +def test_cli_inline_query_stub_has_no_vocab_expansion(): + """cli-inline hosts get the NetworkX-fallback stub, not vocab-expansion.""" + for key in ("codex", "windows"): + core, refs = _platform_artifacts(key) + # The core stub points at the query reference without the vocab step. + assert "expand the question against the graph's own vocabulary" not in core + assert "NetworkX traversal" in core + # The query reference carries the path/explain headings but not the + # claude-only vocab-expansion sub-headings. + q = refs["query.md"] + assert "## For /graphify path" in q + assert "## For /graphify explain" in q + assert "Constrained query expansion" not in q + + +def test_schema_singleton_passes_across_all_platforms(): + """The file_type enum is the six-value superset in every rendered artifact.""" + platforms = gen.load_platforms() + problems = gen.schema_singleton(platforms) + assert problems == [], "\n".join(problems) + + +def test_schema_singleton_catches_legacy_enums(): + """The guard's line scanner flags 4- and 5-value pipe enums, not the superset.""" + four = 'file_type":"code|document|paper|image"' + five = 'file_type":"code|document|paper|image|rationale"' + superset = '"file_type":"code|document|paper|image|rationale|concept"' + assert gen.legacy_enum_lines(four) == [four] + assert gen.legacy_enum_lines(five) == [five] + # The full six-value superset is never flagged. + assert gen.legacy_enum_lines(superset) == [] + assert gen.legacy_enum_lines("no enum here") == [] + + +# --- the remaining progressive hosts ------------------------------------------- + +_PROGRESSIVE_HOSTS = ( + "opencode", + "kilo", + "copilot", + "claw", + "droid", + "amp", + "trae", + "kiro", + "pi", + "vscode", +) + + +def test_all_progressive_hosts_check_and_audit_clean(): + """check + audit-coverage pass for every rendered progressive host.""" + platforms = gen.load_platforms() + for key in _PROGRESSIVE_HOSTS: + arts = gen.render_all(platforms, only=key) + assert gen.check(arts) == [], f"[{key}] check\n" + "\n".join(gen.check(arts)) + probs = gen.audit_coverage(platforms[key]) + assert probs == [], f"[{key}] audit\n" + "\n".join(probs) + + +def test_kiro_and_pi_omit_the_trigger_line(): + """kiro and pi render no frontmatter trigger (trigger absent in v8).""" + for key in ("kiro", "pi"): + core, _ = _platform_artifacts(key) + head = core.split("---", 2)[1] + assert "trigger:" not in head, f"[{key}] unexpectedly has a trigger line" + + +def test_triggered_hosts_keep_the_trigger_line(): + """The other split hosts keep trigger: /graphify in the frontmatter.""" + for key in ("opencode", "kilo", "copilot", "claw", "droid", "amp", "trae", "vscode"): + core, _ = _platform_artifacts(key) + head = core.split("---", 2)[1] + assert "trigger: /graphify" in head, f"[{key}] missing trigger line" + + +def test_kilo_renders_its_rules_tail_section(): + """kilo gets the Kilo-specific rules tail before Honesty Rules.""" + core, _ = _platform_artifacts("kilo") + assert "## Kilo-specific rules" in core + assert core.index("## Kilo-specific rules") < core.index("## Honesty Rules") + + +def test_dispatch_variants_are_host_specific(): + """Each dispatch variant lands in the right host's B2 slot.""" + expect = { + "opencode": "@mention", + "droid": "Task(description=", + "amp": "Task(description=", + "trae": "Task(description=", + "vscode": "paste each response back", + } + for key, marker in expect.items(): + core, _ = _platform_artifacts(key) + b2 = core[core.index("**Step B2"):core.index("**Step B3")] + assert marker.lower() in b2.lower(), f"[{key}] dispatch slot missing {marker!r}" + + +def test_compact_extraction_hosts_use_the_compact_spec(): + """kiro, pi, claw use the compact extraction body; the rest use verbose.""" + for key in ("kiro", "pi", "claw"): + _, refs = _platform_artifacts(key) + assert "(compact)" in refs["extraction-spec.md"], f"[{key}] not compact" + for key in ("opencode", "kilo", "copilot", "droid", "amp", "trae", "vscode"): + _, refs = _platform_artifacts(key) + assert "(compact)" not in refs["extraction-spec.md"], f"[{key}] should be verbose" + + +def test_every_split_host_renders_eight_references(): + """All twelve split hosts render exactly the eight on-demand references.""" + platforms = gen.load_platforms() + expected = [ + "add-watch.md", + "exports.md", + "extraction-spec.md", + "github-and-merge.md", + "hooks.md", + "query.md", + "transcribe.md", + "update.md", + ] + for key, p in platforms.items(): + if p.bucket != "split": + continue + _, refs = _platform_artifacts(key) + assert sorted(refs) == expected, f"[{key}] reference set drift: {sorted(refs)}" + + +# --- the aider + devin monoliths ----------------------------------------------- + + +def test_monoliths_render_inline_single_file_no_references(): + """aider and devin render one inline body, no split and no references dir.""" + platforms = gen.load_platforms() + for key in ("aider", "devin"): + assert platforms[key].bucket == "monolith" + arts = gen.render(platforms[key]) + assert len(arts) == 1, f"[{key}] monolith should render exactly one file" + assert arts[0].path == f"graphify/skill-{key}.md" + assert "references/" not in arts[0].content or "see `references/" not in arts[0].content.lower() + + +def test_monolith_roundtrip_passes_for_aider_and_devin(): + """Each monolith is diff-clean vs v8 except the file_type enum unification.""" + platforms = gen.load_platforms() + for key in ("aider", "devin"): + problems = gen.monolith_roundtrip(platforms[key]) + assert problems == [], f"[{key}]\n" + "\n".join(problems) + + +def test_monoliths_change_only_the_enum_and_the_description(): + """The rendered monolith differs from v8 on exactly the enum + description lines. + + Two changes are now in play for the monoliths: the file_type enum unified to + the six-value superset (the prose guidance line + the schema line) and the + frontmatter description unified across all platforms. Nothing else may differ. + """ + platforms = gen.load_platforms() + for key in ("aider", "devin"): + rendered = gen.render(platforms[key])[0].content.splitlines() + original = gen._normalise(gen._git_show(platforms[key].roundtrip_ref)).splitlines() + assert len(rendered) == len(original), f"[{key}] line count changed" + diff_idx = [i for i, (r, o) in enumerate(zip(rendered, original)) if r != o] + # Exactly three lines change: the prose enum guidance, the schema line, + # and the frontmatter description. + assert len(diff_idx) == 3, f"[{key}] expected 3 changed lines, got {len(diff_idx)}" + enum_changes = 0 + desc_changes = 0 + for i in diff_idx: + line = rendered[i] + if gen.ENUM_VALUES in line or gen.ENUM_PROSE in line: + enum_changes += 1 + elif line.lstrip().startswith("description:"): + desc_changes += 1 + assert UNIFIED_DESCRIPTION in line, ( + f"[{key}] description line is not the unified text: {line!r}" + ) + else: + raise AssertionError( + f"[{key}] changed line {i} is neither enum nor description: {line!r}" + ) + assert enum_changes == 2, f"[{key}] expected 2 enum line changes, got {enum_changes}" + assert desc_changes == 1, f"[{key}] expected 1 description change, got {desc_changes}" + # The six-value superset replaced the five-value enum in both files. + assert any(gen.ENUM_VALUES in line for line in rendered) + + +def test_devin_keeps_its_multi_field_frontmatter(): + """devin renders inline, so its 4+-field frontmatter is preserved verbatim.""" + platforms = gen.load_platforms() + body = gen.render(platforms["devin"])[0].content + head = body.split("---", 2)[1] + assert "argument-hint:" in head + assert "model:" in head + assert "allowed-tools:" in head + + +# --- the always-on instruction blocks (D2-a) ----------------------------------- + + +def test_always_on_renders_six_blocks(): + """render_always_on yields exactly the six always-on instruction files.""" + arts = gen.render_always_on() + paths = sorted(a.path for a in arts) + assert paths == [ + "graphify/always_on/agents-md.md", + "graphify/always_on/antigravity-rules.md", + "graphify/always_on/claude-md.md", + "graphify/always_on/gemini-md.md", + "graphify/always_on/kiro-steering.md", + "graphify/always_on/vscode-instructions.md", + ] + + +def test_always_on_included_in_full_render_not_per_platform(): + """A full render carries the always-on files; a --platform render does not.""" + platforms = gen.load_platforms() + full = {a.path for a in gen.render_all(platforms)} + claude_only = {a.path for a in gen.render_all(platforms, only="claude")} + assert "graphify/always_on/claude-md.md" in full + assert "graphify/always_on/claude-md.md" not in claude_only + + +def test_always_on_roundtrip_is_byte_faithful(): + """Each always_on/*.md reproduces its former __main__.py constant byte for byte. + + This is the load-bearing fidelity check behind the D2-a extraction: the + install-string / issue-#580 tests still import the constants from + graphify.__main__, so the packaged markdown must round-trip exactly or those + contracts silently change. + """ + problems = gen.always_on_roundtrip() + assert problems == [], "\n".join(problems) + + +def test_extracted_constants_equal_the_packaged_always_on_files(): + """The live module constants now equal the packaged files they read at load.""" + from graphify import __main__ as mainmod + + pairs = { + "_CLAUDE_MD_SECTION": "claude-md", + "_AGENTS_MD_SECTION": "agents-md", + "_GEMINI_MD_SECTION": "gemini-md", + "_VSCODE_INSTRUCTIONS_SECTION": "vscode-instructions", + "_ANTIGRAVITY_RULES": "antigravity-rules", + "_KIRO_STEERING": "kiro-steering", + } + pkg = Path(mainmod.__file__).parent + for const_name, basename in pairs.items(): + on_disk = (pkg / "always_on" / f"{basename}.md").read_text(encoding="utf-8") + assert getattr(mainmod, const_name) == on_disk, const_name + + +def test_always_on_files_are_guarded_by_check(tmp_path): + """A hand-edit of an always_on/*.md is caught by --check (the drift guard).""" + platforms = gen.load_platforms() + arts = gen.render_all(platforms) + # The committed + expected/ snapshots match a fresh render. + assert gen.check(arts) == [], "\n".join(gen.check(arts)) + # A mutated artifact is flagged. + mutated = [ + gen.RenderedArtifact(a.path, a.content + "drift\n") + if a.path == "graphify/always_on/claude-md.md" + else a + for a in arts + ] + problems = gen.check(mutated) + assert any("always_on/claude-md.md" in p for p in problems) + + +# --- the per-host coverage audit (the systemic guard) -------------------------- + + +def test_audit_coverage_passes_for_every_split_host(): + """Every split host's render single-homes its own v8 body's headings.""" + platforms = gen.load_platforms() + for key, p in platforms.items(): + if p.bucket != "split": + continue + problems = gen.audit_coverage(p) + assert problems == [], f"[{key}]\n" + "\n".join(problems) + + +def test_audit_reads_each_host_against_its_own_v8_body(): + """The audit baseline is the host's OWN v8 skill body, not claude's monolith. + + This is the structural fix: a per-host body, so a drop on one host surfaces. + """ + assert gen._v8_baseline_ref("claude") == "origin/v8:graphify/skill.md" + assert gen._v8_baseline_ref("trae") == "origin/v8:graphify/skill-trae.md" + assert gen._v8_baseline_ref("vscode") == "origin/v8:graphify/skill-vscode.md" + + +def test_audit_catches_an_induced_per_host_drop(): + """Re-inducing the trae regression (claude-flavored hooks) fails the audit. + + Pointing trae back at the shared CLAUDE.md hooks body drops the + '## For native AGENTS.md integration (Trae)' heading from its render. The + per-host audit must catch that against trae's own v8 body. The old audit + (every host vs claude's monolith) could not see it, because claude's monolith + never had that heading. + """ + import dataclasses + + platforms = gen.load_platforms() + regressed = dataclasses.replace(platforms["trae"], hooks_variant="claude-md") + problems = gen.audit_coverage(regressed) + assert any("native AGENTS.md integration (Trae)" in p for p in problems), problems + + +def test_audit_catches_a_dropped_non_allowlisted_heading(): + """A core fragment that drops a real v8 heading fails the audit. + + Guards that the audit is not a rubber stamp: a host whose v8 has a heading + that is neither allowlisted nor present anywhere in the render must fail. + """ + platforms = gen.load_platforms() + trae = platforms["trae"] + real_arts = gen.render(trae) + # Drop the Honesty Rules heading from the rendered core to simulate a real + # content loss, then re-run the single-home check by hand against trae's v8. + v8_headings = gen.headings(gen._git_show(gen._v8_baseline_ref("trae"))) + assert "## Honesty Rules" in v8_headings + by_path = {a.path: a.content for a in real_arts} + core_no_honesty = by_path[trae.skill_dst].replace("## Honesty Rules", "## Closing notes") + core_headings = set(gen.headings(core_no_honesty)) + allowlist = gen._audit_allowlist("trae") + homes = [h for h in v8_headings if h == "## Honesty Rules" and h in core_headings] + assert "## Honesty Rules" not in allowlist + assert homes == [], "a dropped, non-allowlisted heading should have no home" + + +def test_git_show_validators_skip_cleanly_without_origin_v8(monkeypatch, tmp_path, capsys): + """On a shallow checkout (no origin/v8) the validators skip with exit 0. + + CI sets fetch-depth: 0 so they run for real; this guards the fallback so a + shallow clone gets a clear message instead of a crash. + """ + import subprocess as sp + + repo = tmp_path / "shallow" + repo.mkdir() + sp.run(["git", "init", "-q", str(repo)], check=True) + monkeypatch.setattr(gen, "REPO_ROOT", repo) + assert gen._v8_available() is False + for flag in ("--audit-coverage", "--monolith-roundtrip", "--always-on-roundtrip"): + assert gen.main([flag]) == 0 + out = capsys.readouterr() + assert "SKIPPED" in out.err + assert "fetch-depth: 0" in out.err + + +def test_audit_allowlist_documents_only_consolidations(): + """The allowlist holds only the wave-2/3 consolidations, nothing genuine. + + A genuine drop (trae's native AGENTS.md integration) must never be in the + allowlist, or the guard would rubber-stamp the regression it exists to catch. + """ + all_allowlisted = set(gen.SHARED_INTRO_ALLOWLIST) + for hs in gen._CONSOLIDATION_ALLOWLIST.values(): + all_allowlisted |= set(hs) + assert "## For native AGENTS.md integration (Trae)" not in all_allowlisted + # Only the two minimal-body hosts carry per-host consolidations. + assert set(gen._CONSOLIDATION_ALLOWLIST) == {"kilo", "vscode"} + + +# --- the trae / trae-cn native AGENTS.md integration fix ----------------------- + + +def test_trae_renders_native_agents_md_integration_not_claude(): + """trae wires `graphify trae install` -> AGENTS.md, never `graphify claude install`.""" + core, refs = _platform_artifacts("trae") + hooks = refs["hooks.md"] + # The hooks reference carries the v8 native AGENTS.md integration section. + assert "## For native AGENTS.md integration (Trae)" in hooks + assert "graphify trae install" in hooks + assert "graphify trae-cn install" in hooks + assert "writes a `## graphify` section to the local `AGENTS.md`" in hooks + # The claude-flavored install command must NOT appear for trae. + assert "graphify claude install" not in hooks + assert "native CLAUDE.md integration" not in hooks + # The lean-core pointer names AGENTS.md, not CLAUDE.md. + assert "## For the commit hook and native AGENTS.md integration" in core + assert "wire graphify into a project's AGENTS.md" in core + assert "native CLAUDE.md integration" not in core + + +def test_trae_dispatch_carries_the_no_pretooluse_caveat(): + """trae's B2 dispatch block restores the v8 no-PreToolUse-hook caveat.""" + core, _ = _platform_artifacts("trae") + b2 = core[core.index("**Step B2"):core.index("Pass the extraction prompt")] + assert "Trae does NOT support PreToolUse hooks" in b2 + assert "AGENTS.md rules are the always-on mechanism instead" in b2 + + +def test_trae_hooks_reference_includes_the_pretooluse_note(): + """The trae hooks reference keeps the v8 PreToolUse note in full.""" + _, refs = _platform_artifacts("trae") + hooks = refs["hooks.md"] + assert "Unlike Claude Code, Trae does NOT support PreToolUse hooks" in hooks + assert "Run `/graphify --update` manually after code changes" in hooks + + +def test_claude_flavored_hosts_keep_their_hooks_text_unchanged(): + """Hosts whose v8 shipped the claude-flavored hooks keep it (faithful to them). + + droid's v8 dispatch never had the Trae caveat and its hooks section names + CLAUDE.md; restoring trae must not bleed into droid or any other host. + """ + for key in ("claude", "droid", "codex", "windows", "kilo", "vscode"): + core, refs = _platform_artifacts(key) + hooks = refs["hooks.md"] + assert "graphify claude install" in hooks, f"[{key}] lost the claude install command" + assert "native CLAUDE.md integration" in hooks, f"[{key}] lost the CLAUDE.md heading" + assert "Trae does NOT support PreToolUse hooks" not in core, f"[{key}] leaked the trae caveat" + assert "Trae does NOT support PreToolUse hooks" not in hooks, f"[{key}] leaked the trae caveat" + assert "## For the commit hook and native CLAUDE.md integration" in core, f"[{key}] pointer drifted" + + +# --- the amp native AGENTS.md integration (the 13th split host) ---------------- + + +def test_amp_renders_native_agents_md_integration_v8_faithfully(): + """amp wires `graphify amp install` -> AGENTS.md exactly as its v8 body had it. + + amp shares the agents-md hooks variant with trae but renders its OWN wording: + a bare "## For native AGENTS.md integration" heading (no "(Trae)" suffix), + single-line install/uninstall commands (no trae-cn alt), and crucially NO + PreToolUse caveat (amp's v8 never carried one). + """ + core, refs = _platform_artifacts("amp") + hooks = refs["hooks.md"] + # amp's bare v8 heading and Amp-worded prose. + assert "## For native AGENTS.md integration" in hooks + assert "## For native AGENTS.md integration (Trae)" not in hooks + assert "make graphify always-on in Amp sessions" in hooks + assert "instructs Amp to check the graph" in hooks + # amp's single-line install/uninstall, no trae-cn alt comments. + assert "graphify amp install" in hooks + assert "graphify amp uninstall # remove the section" in hooks + assert "graphify trae install" not in hooks + assert "graphify trae-cn" not in hooks + assert "or: graphify" not in hooks + # No claude flavoring on amp. + assert "graphify claude install" not in hooks + assert "native CLAUDE.md integration" not in hooks + # The lean-core pointer names AGENTS.md, not CLAUDE.md. + assert "## For the commit hook and native AGENTS.md integration" in core + assert "wire graphify into a project's AGENTS.md" in core + assert "native CLAUDE.md integration" not in core + + +def test_amp_has_no_pretooluse_caveat_anywhere(): + """amp's v8 had no no-PreToolUse-hooks note, so neither its core nor hooks may. + + This is the explicit guard against injecting trae-specific wording into amp. + The caveat belongs to trae alone; amp uses the plain task-tool-disk dispatch + and a caveat-free AGENTS.md integration section. + """ + core, refs = _platform_artifacts("amp") + hooks = refs["hooks.md"] + assert "PreToolUse" not in core, "amp leaked a PreToolUse caveat into its core" + assert "PreToolUse" not in hooks, "amp leaked a PreToolUse caveat into its hooks reference" + assert "Trae does NOT support" not in core + assert "Trae does NOT support" not in hooks + # amp's dispatch is the plain task-tool-disk block (no trae caveat line). + b2 = core[core.index("**Step B2"):core.index("Pass the extraction prompt")] + assert "Trae" not in b2 + + +def test_amp_audit_coverage_passes_against_its_own_v8(): + """The per-host audit (the guard amp is the exact case for) passes for amp. + + amp was omitted from wave 3's render list, so its v8 body was never audited + against a lean split. The audit reads origin/v8:graphify/skill-amp.md and + confirms every heading single-homes in amp's core + references. + """ + platforms = gen.load_platforms() + assert gen._v8_baseline_ref("amp") == "origin/v8:graphify/skill-amp.md" + problems = gen.audit_coverage(platforms["amp"]) + assert problems == [], "\n".join(problems) diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/skillgen/__init__.py b/tools/skillgen/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/skillgen/__main__.py b/tools/skillgen/__main__.py new file mode 100644 index 00000000..c4c741af --- /dev/null +++ b/tools/skillgen/__main__.py @@ -0,0 +1,7 @@ +"""Entry point for ``python -m tools.skillgen``.""" +from __future__ import annotations + +from tools.skillgen.gen import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/skillgen/expected/graphify__always_on__agents-md.md b/tools/skillgen/expected/graphify__always_on__agents-md.md new file mode 100644 index 00000000..20cff728 --- /dev/null +++ b/tools/skillgen/expected/graphify__always_on__agents-md.md @@ -0,0 +1,12 @@ +## graphify + +This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. + +When the user types `/graphify`, invoke the `skill` tool with `skill: "graphify"` before doing anything else. + +Rules: +- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- Dirty graphify-out/ files are expected after hooks or incremental updates; dirty graph files are not a reason to skip graphify. Only skip graphify if the task is about stale or incorrect graph output, or the user explicitly says not to use it. +- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. +- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/tools/skillgen/expected/graphify__always_on__antigravity-rules.md b/tools/skillgen/expected/graphify__always_on__antigravity-rules.md new file mode 100644 index 00000000..0fc78641 --- /dev/null +++ b/tools/skillgen/expected/graphify__always_on__antigravity-rules.md @@ -0,0 +1,14 @@ +--- +trigger: always_on +description: Consult the graphify knowledge graph at graphify-out/ for codebase and architecture questions. +--- + +## graphify + +This project has a graphify knowledge graph at graphify-out/. + +Rules: +- For codebase or architecture questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (CLI) or `query_graph` (MCP). Use `graphify path "" ""` / `shortest_path` for relationships and `graphify explain ""` / `get_node` for focused concepts. These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. +- If graphify-out/wiki/index.md exists, navigate it instead of reading raw files +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context +- After modifying code files in this session, run `graphify update .` to keep the graph current (AST-only, no API cost) diff --git a/tools/skillgen/expected/graphify__always_on__claude-md.md b/tools/skillgen/expected/graphify__always_on__claude-md.md new file mode 100644 index 00000000..417efeb2 --- /dev/null +++ b/tools/skillgen/expected/graphify__always_on__claude-md.md @@ -0,0 +1,9 @@ +## graphify + +This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. + +Rules: +- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. +- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/tools/skillgen/expected/graphify__always_on__gemini-md.md b/tools/skillgen/expected/graphify__always_on__gemini-md.md new file mode 100644 index 00000000..417efeb2 --- /dev/null +++ b/tools/skillgen/expected/graphify__always_on__gemini-md.md @@ -0,0 +1,9 @@ +## graphify + +This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. + +Rules: +- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. +- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/tools/skillgen/expected/graphify__always_on__kiro-steering.md b/tools/skillgen/expected/graphify__always_on__kiro-steering.md new file mode 100644 index 00000000..cb6f4543 --- /dev/null +++ b/tools/skillgen/expected/graphify__always_on__kiro-steering.md @@ -0,0 +1,5 @@ +--- +inclusion: always +--- + +graphify: A knowledge graph of this project lives in `graphify-out/`. For codebase, architecture, or dependency questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. Read `GRAPH_REPORT.md` only for broad architecture review or when those commands do not surface enough context. diff --git a/tools/skillgen/expected/graphify__always_on__vscode-instructions.md b/tools/skillgen/expected/graphify__always_on__vscode-instructions.md new file mode 100644 index 00000000..9cb983c9 --- /dev/null +++ b/tools/skillgen/expected/graphify__always_on__vscode-instructions.md @@ -0,0 +1,17 @@ +## graphify + +For any question about this repo's architecture, structure, components, or how to add/modify/find +code, your first action should be `graphify query ""` when `graphify-out/graph.json` +exists. Use `graphify path "" ""` for relationship questions and `graphify explain ""` +for focused-concept questions. These return a scoped subgraph, usually much smaller than the full +report or raw grep output. + +Triggers: "how do I…", "where is…", "what does … do", "add/modify a ", +"explain the architecture", or anything that depends on how files or classes relate. + +If `graphify-out/wiki/index.md` exists, use it for broad navigation. Read `graphify-out/GRAPH_REPORT.md` +only for broad architecture review or when query/path/explain do not surface enough context. Only read +source files when (a) modifying/debugging specific code, (b) the graph lacks the needed detail, or +(c) the graph is missing or stale. + +Type `/graphify` in Copilot Chat to build or update the graph. diff --git a/tools/skillgen/expected/graphify__skill-aider.md b/tools/skillgen/expected/graphify__skill-aider.md new file mode 100644 index 00000000..aad4ff95 --- /dev/null +++ b/tools/skillgen/expected/graphify__skill-aider.md @@ -0,0 +1,1246 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. + +Three things it does that your AI assistant alone cannot: +1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. +2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. +3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. + +Use it for: +- A codebase you're new to (understand architecture before touching anything) +- A reading list (papers + tweets + notes → one navigable graph) +- A research corpus (citation graph + concept graph in one) +- Your personal /raw folder (drop everything in, let it grow, query it) + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +Follow these steps in order. Do not skip steps. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result)) +" > .graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. + +**Step 2 - Transcribe:** + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('.graphify_detect.json').read_text()) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files) + Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +> **Aider platform:** Multi-agent support is still early on Aider. Extraction runs sequentially — you read and extract each file yourself. This is slower than parallel platforms but fully reliable. + +Print: `"Semantic extraction: N files (sequential — Aider)"` + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('.graphify_detect.json').read_text()) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) +Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Sequential extraction (Aider)** + +Process each file one at a time. For each file: + +1. Read the file contents +2. Extract nodes, edges, and hyperedges applying the same rules: + - EXTRACTED: relationship explicit in source (import, call, citation) + - INFERRED: reasonable inference (shared structure, implied dependency) + - AMBIGUOUS: uncertain — flag it, do not omit + - Code files: semantic edges AST cannot find. Do not re-extract imports. + - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. When adding `calls` edges: source is caller, target is callee. + - Image files: use vision — understand what the image IS, not just OCR + - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges + - Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. + - Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. + - confidence_score REQUIRED on every edge: EXTRACTED=1.0, INFERRED=0.6-0.9 (reason individually), AMBIGUOUS=0.1-0.3 +3. Accumulate results across all files + +Schema for each file's output: +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +After processing all files, write the accumulated result to `.graphify_semantic_new.json`. + +**Step B3 - Cache and merge** + +For the accumulated result: + +If more than half the chunks failed, stop and tell the user. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text()) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2)) +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('.graphify_ast.json').read_text()) +sem = json.loads(Path('.graphify_semantic.json').read_text()) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +detection = json.loads(Path('.graphify_detect.json').read_text()) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +detection = json.loads(Path('.graphify_detect.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_obsidian, to_canvas +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) +print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') + +to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) +print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') +print() +print('Open graphify-out/obsidian/ as a vault in Obsidian.') +print(' Graph view - nodes colored by community (set automatically)') +print(' graph.canvas - structured layout with communities as groups') +print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') +" +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_html +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +if G.number_of_nodes() > 5000: + print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') +else: + to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) + print('graph.html written - open in any browser, no server needed') +" +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_cypher +from pathlib import Path + +G = build_from_json(json.loads(Path('.graphify_extract.json').read_text())) +to_cypher(G, 'graphify-out/cypher.txt') +print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') +" +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster +from graphify.export import push_to_neo4j +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') +" +``` + +Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_svg +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) +print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') +" +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.build import build_from_json +from graphify.export import to_graphml +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +to_graphml(G, communities, 'graphify-out/graph.graphml') +print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') +" +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `.graphify_detect.json` is greater than 5,000, run: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.benchmark import run_benchmark, print_benchmark +from pathlib import Path + +detection = json.loads(Path('.graphify_detect.json').read_text()) +result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) +print_benchmark(result) +" +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('.graphify_detect.json').read_text()) +save_manifest(detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('.graphify_extract.json').read_text()) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text()) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2)) + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2)) +Path('.graphify_incremental.json').write_text(json.dumps(result)) +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load existing graph +existing_data = json.loads(Path('graphify-out/graph.json').read_text()) +G_existing = json_graph.node_link_graph(existing_data, edges='links') + +# Load new extraction +new_extraction = json.loads(Path('.graphify_extract.json').read_text()) +G_new = build_from_json(new_extraction) + +# Merge: new nodes/edges into existing graph +G_existing.update(G_new) +print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None +new_extract = json.loads(Path('.graphify_extract.json').read_text()) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` +Clean up after: `rm -f .graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections +from graphify.report import generate +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, + 'files': {'code': [], 'document': [], 'paper': []}} +tokens = {'input': 0, 'output': 0} + +communities = cluster(G) +cohesion = score_all(G, communities) +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, +} +Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +print(f'Re-clustered: {len(communities)} communities') +" +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). + +--- + +## For /graphify query + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Load `graphify-out/graph.json`, then: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` + +--- + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. + +--- + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md new file mode 100644 index 00000000..9cad76fc --- /dev/null +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -0,0 +1,612 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + +Follow these steps in order. Do not skip steps. + +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Video and audio (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch ALL subagents in a single message** + +> Uses the `Task` tool for parallel subagent dispatch. +> Call `Task` once per chunk — ALL in the same response so they run in parallel. + +Pass the extraction prompt as the task description: + +``` +Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") +``` + +Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. + +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi +``` + +## For --update and --cluster-only + +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: + +```bash +graphify query "" +``` + +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native AGENTS.md integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's AGENTS.md, see `references/hooks.md`. + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md new file mode 100644 index 00000000..bad0a0d5 --- /dev/null +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -0,0 +1,615 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + +Follow these steps in order. Do not skip steps. + +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Video and audio (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch ALL subagents in a single message** + +Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. + +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. + +Concrete example for 3 chunks: +``` +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] +``` +All three in one message. Not three separate messages. + +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi +``` + +## For --update and --cluster-only + +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: + +```bash +graphify query "" +``` + +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native CLAUDE.md integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md new file mode 100644 index 00000000..b9a70f06 --- /dev/null +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -0,0 +1,612 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + +Follow these steps in order. Do not skip steps. + +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Video and audio (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch ALL subagents in a single message (Codex)** + +> **Codex platform:** Uses `spawn_agent` + `wait_agent` + `close_agent` instead of the Agent tool. +> Requires `multi_agent = true` under `[features]` in `~/.codex/config.toml`. +> If `spawn_agent` is unavailable, tell the user to add that config and restart Codex. + +Call `spawn_agent` once per chunk — ALL in the same response so they run in parallel. Build the message by wrapping the extraction prompt in task-delegation framing: + +``` +spawn_agent(agent_type="worker", message="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") +``` + +After all agents are dispatched, collect results sequentially in memory: +``` +result = wait_agent(handle); close_agent(handle) # repeat per handle +``` + +Parse each result as JSON. Accumulate nodes/edges/hyperedges across all results and write to `graphify-out/.graphify_semantic_new.json`. Codex collects in memory, so there are no per-chunk files on disk; the disk-based success checks in Step B3 do not apply — a chunk that returns invalid JSON is the failure signal instead. + +Subagent prompt template: + +See `references/extraction-spec.md` for the compact subagent prompt (rules, node-ID format, confidence rubric, hyperedge and vision rules, JSON schema). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each agent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted, and have it return the JSON inline. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi +``` + +## For --update and --cluster-only + +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: + +```bash +graphify query "" +``` + +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native CLAUDE.md integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md new file mode 100644 index 00000000..bad0a0d5 --- /dev/null +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -0,0 +1,615 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + +Follow these steps in order. Do not skip steps. + +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Video and audio (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch ALL subagents in a single message** + +Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. + +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. + +Concrete example for 3 chunks: +``` +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] +``` +All three in one message. Not three separate messages. + +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi +``` + +## For --update and --cluster-only + +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: + +```bash +graphify query "" +``` + +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native CLAUDE.md integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tools/skillgen/expected/graphify__skill-devin.md b/tools/skillgen/expected/graphify__skill-devin.md new file mode 100644 index 00000000..7013159a --- /dev/null +++ b/tools/skillgen/expected/graphify__skill-devin.md @@ -0,0 +1,1372 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +argument-hint: "[path|query|subcommand]" +model: sonnet +allowed-tools: + - read + - grep + - glob + - exec +triggers: + - user + - model +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory +/graphify # full pipeline on specific path +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. + +Three things it does that your AI assistant alone cannot: +1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. +2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. +3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. + +Use it for: +- A codebase you're new to (understand architecture before touching anything) +- A reading list (papers + tweets + notes -> one navigable graph) +- A research corpus (citation graph + concept graph in one) +- Your personal /raw folder (drop everything in, let it grow, query it) + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +Follow these steps in order. Do not skip steps. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Force UTF-8 I/O on Windows (prevents garbled CJK/non-ASCII output) +export PYTHONUTF8=1 +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. + +**Step 2 - Transcribe:** + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2)) + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**MANDATORY: You MUST use the subagent system here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use parallel subagents you are doing this wrong.** + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached)) +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch subagents** + +Dispatch ALL subagents in the same response so they run in parallel. + +Concrete example for 3 chunks: +``` +[Subagent 1: files 1-15] +[Subagent 2: files 16-30] +[Subagent 3: files 31-45] +``` +All three in one message. Not three separate messages. + +For each chunk, dispatch a subagent with this exact prompt (fill in FILE_LIST): + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON with no commentary: {"nodes": [...], "edges": [...], "hyperedges": [...], "input_tokens": 0, "output_tokens": 0} + +Extraction rules: +- EXTRACTED: relationship explicit in source (import, call, citation) +- INFERRED: reasonable inference (shared structure, implied dependency) +- AMBIGUOUS: uncertain — flag it, do not omit +- Code files: extract semantic edges AST cannot find (design patterns, protocol conformance). Do not re-extract imports. +- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. When adding `calls` edges: source is caller, target is callee. +- Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. +- DEEP_MODE (if set): be aggressive with INFERRED edges +- Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. +- Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. +- If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. +- confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Schema: +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Files: +FILE_LIST +``` + +**Step B3 - Cache and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_N.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +After each subagent call completes, write its result to `graphify-out/.graphify_chunk_N.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then merge: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path +from graphify.semantic_cleanup import load_validated_semantic_fragment, sanitize_semantic_fragment + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d, errors = load_validated_semantic_fragment(Path(c)) + if errors: + print(f'Skipping invalid chunk {c}: ' + '; '.join(errors[:3])) + continue + d = sanitize_semantic_fragment(d) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2)) +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.semantic_cleanup import sanitize_semantic_fragment + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text()) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +merged = sanitize_semantic_fragment(merged) +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path +from graphify.semantic_cleanup import sanitize_semantic_fragment + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text()) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text()) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +merged = sanitize_semantic_fragment(merged) +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_obsidian, to_canvas +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) +print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') + +to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) +print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') +print() +print('Open graphify-out/obsidian/ as a vault in Obsidian.') +print(' Graph view - nodes colored by community (set automatically)') +print(' graph.canvas - structured layout with communities as groups') +print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') +" +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_html +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +NODE_LIMIT = 5000 +if G.number_of_nodes() > NODE_LIMIT: + from collections import Counter + print(f'Graph has {G.number_of_nodes()} nodes (above {NODE_LIMIT} limit). Building aggregated community view...') + node_to_community = {nid: cid for cid, members in communities.items() for nid in members} + import networkx as nx_meta + meta = nx_meta.Graph() + for cid, members in communities.items(): + meta.add_node(str(cid), label=labels.get(cid, f'Community {cid}')) + edge_counts = Counter() + for u, v in G.edges(): + cu, cv = node_to_community.get(u), node_to_community.get(v) + if cu is not None and cv is not None and cu != cv: + edge_counts[(min(cu, cv), max(cu, cv))] += 1 + for (cu, cv), w in edge_counts.items(): + meta.add_edge(str(cu), str(cv), weight=w, relation=f'{w} cross-community edges', confidence='AGGREGATED') + if meta.number_of_nodes() > 1: + meta_communities = {cid: [str(cid)] for cid in communities} + member_counts = {cid: len(members) for cid, members in communities.items()} + to_html(meta, meta_communities, 'graphify-out/graph.html', community_labels=labels or None, member_counts=member_counts) + print(f'graph.html written (aggregated: {meta.number_of_nodes()} community nodes, {meta.number_of_edges()} cross-community edges)') + print('Tip: run with --obsidian for full node-level detail, or --wiki for an agent-crawlable wiki.') + else: + print('Single community — aggregated view not useful. Skipping graph.html.') +else: + to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) + print('graph.html written - open in any browser, no server needed') +" +``` + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +The wiki is an agent-crawlable export — `index.md` plus one article per community plus god-node articles. It is the recommended fallback for graphs too large to render as HTML, and it's the most useful output for an autonomous agent navigating the graph between sessions. + +Run this before Step 9 (cleanup) so `graphify-out/.graphify_labels.json` is still available. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.build import build_from_json +from graphify.wiki import to_wiki +from graphify.analyze import god_nodes +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +labels = {int(k): v for k, v in labels_raw.items()} +gods = god_nodes(G) + +n = to_wiki(G, communities, 'graphify-out/wiki', community_labels=labels or None, cohesion=cohesion, god_nodes_data=gods) +print(f'Wiki: {n} articles written to graphify-out/wiki/') +print(' graphify-out/wiki/index.md -> agent entry point') +" +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_cypher +from pathlib import Path + +G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text())) +to_cypher(G, 'graphify-out/cypher.txt') +print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') +" +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import push_to_neo4j +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') +" +``` + +Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_svg +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) +print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') +" +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.build import build_from_json +from graphify.export import to_graphml +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +to_graphml(G, communities, 'graphify-out/graph.graphml') +print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') +" +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.benchmark import run_benchmark, print_benchmark +from pathlib import Path + +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) +print_benchmark(result) +" +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +save_manifest(detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text()) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2)) + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json graphify-out/.graphify_chunk_*.json graphify-out/.graphify_incremental.json graphify-out/.graphify_transcripts.json graphify-out/.graphify_old.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given; omit the wiki line unless --wiki was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) + wiki/ - agent-crawlable wiki, start at wiki/index.md (only if --wiki was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" +fi +``` + +--- + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result)) +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4-8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A-3C pipeline as normal. + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load existing graph +existing_data = json.loads(Path('graphify-out/graph.json').read_text()) +G_existing = json_graph.node_link_graph(existing_data, edges='links') + +# Load new extraction +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +G_new = build_from_json(new_extraction) + +# Merge: new nodes/edges into existing graph +G_existing.update(G_new) +print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') +" +``` + +Then run Steps 4-8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text()) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1-3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections +from graphify.report import generate +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, + 'files': {'code': [], 'document': [], 'paper': []}} +tokens = {'input': 0, 'output': 0} + +communities = cluster(G) +cohesion = score_all(G, communities) +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +print(f'Re-clustered: {len(communities)} communities') +" +``` + +Then run Steps 5-9 as normal (label communities, generate viz, benchmark, clean up, report). + +--- + +## For /graphify query + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Load `graphify-out/graph.json`, then: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept. Then write a 3-5 sentence explanation using source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` + +--- + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- Twitter/X -> fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv -> abstract + metadata saved as `.md` +- PDF -> downloaded as `.pdf` +- Images (.png/.jpg/.webp) -> downloaded, vision extraction runs on next build +- Any webpage -> converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +--- + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed, re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +--- + +## For always-on context in Devin sessions + +Run once per project to make graphify always-on in Devin sessions: + +```bash +graphify devin install --project +``` + +This writes a `## graphify` section to `.windsurf/rules/graphify.md` that instructs Devin to check the graph before answering codebase questions and rebuild it after code changes. + +```bash +graphify devin uninstall --project # remove +``` + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md new file mode 100644 index 00000000..6c965c67 --- /dev/null +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -0,0 +1,612 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + +Follow these steps in order. Do not skip steps. + +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Video and audio (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch ALL subagents in a single message** + +> Uses the `Task` tool for parallel subagent dispatch. +> Call `Task` once per chunk — ALL in the same response so they run in parallel. + +Pass the extraction prompt as the task description: + +``` +Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") +``` + +Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. + +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi +``` + +## For --update and --cluster-only + +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: + +```bash +graphify query "" +``` + +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native CLAUDE.md integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md new file mode 100644 index 00000000..8bc6d536 --- /dev/null +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -0,0 +1,624 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + +Follow these steps in order. Do not skip steps. + +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Video and audio (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch ALL subagents in a single message** + +Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. + +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. + +Concrete example for 3 chunks: +``` +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] +``` +All three in one message. Not three separate messages. + +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi +``` + +## For --update and --cluster-only + +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: + +```bash +graphify query "" +``` + +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native CLAUDE.md integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. + +--- + +## Kilo-specific rules + +- Use the native `Task` tool for semantic extraction fan-out. +- Launch all chunk tasks in the same response so they run in parallel. +- Always use `subagent_type="general"` for extraction chunks. +- After modifying code files during the session, run `graphify update .`. + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md new file mode 100644 index 00000000..796ad8ca --- /dev/null +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -0,0 +1,614 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + +Follow these steps in order. Do not skip steps. + +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Video and audio (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch ALL subagents in a single message** + +Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. + +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. + +Concrete example for 3 chunks: +``` +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] +``` +All three in one message. Not three separate messages. + +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi +``` + +## For --update and --cluster-only + +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: + +```bash +graphify query "" +``` + +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native CLAUDE.md integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md new file mode 100644 index 00000000..6c4f2086 --- /dev/null +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -0,0 +1,607 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + +Follow these steps in order. Do not skip steps. + +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Video and audio (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch ALL subagents in a single message (OpenCode)** + +> **OpenCode platform:** Uses `@mention` dispatch instead of the Agent tool. All mentions in a single message run in parallel. + +Dispatch one `@mention` per chunk — ALL in the same response: + +``` +@agent Chunk CHUNK_NUM of TOTAL_CHUNKS: [extraction prompt with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted] + +@agent Chunk 2 of TOTAL_CHUNKS: [next chunk] +``` + +Wait for all agents to return. Parse each response as JSON. Accumulate nodes/edges/hyperedges across all results and write to `graphify-out/.graphify_semantic_new.json`. If the `@agent` path cannot write chunk files, fall back to the serial path that writes each `graphify-out/.graphify_chunk_NN.json` before merge. + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each agent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi +``` + +## For --update and --cluster-only + +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: + +```bash +graphify query "" +``` + +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native CLAUDE.md integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md new file mode 100644 index 00000000..796ad8ca --- /dev/null +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -0,0 +1,614 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + +Follow these steps in order. Do not skip steps. + +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Video and audio (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch ALL subagents in a single message** + +Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. + +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. + +Concrete example for 3 chunks: +``` +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] +``` +All three in one message. Not three separate messages. + +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi +``` + +## For --update and --cluster-only + +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: + +```bash +graphify query "" +``` + +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native CLAUDE.md integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md new file mode 100644 index 00000000..8ba7c66f --- /dev/null +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -0,0 +1,613 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + +Follow these steps in order. Do not skip steps. + +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Video and audio (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch ALL subagents in a single message** + +> Uses the `Task` tool for parallel subagent dispatch. +> Call `Task` once per chunk — ALL in the same response so they run in parallel. +> Trae does NOT support PreToolUse hooks — AGENTS.md rules are the always-on mechanism instead. + +Pass the extraction prompt as the task description: + +``` +Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") +``` + +Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. + +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi +``` + +## For --update and --cluster-only + +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: + +```bash +graphify query "" +``` + +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native AGENTS.md integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's AGENTS.md, see `references/hooks.md`. + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md new file mode 100644 index 00000000..7be667ca --- /dev/null +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -0,0 +1,611 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + +Follow these steps in order. Do not skip steps. + +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Video and audio (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch subagents and paste their responses** + +> **No automated subagent tool:** this host has no parallel Agent/Task API, so +> extraction is driven by hand. Dispatch a subagent per chunk however the host +> allows (a fresh conversation, a parallel pane), then paste each response back. + +For each chunk of uncached files (20-25 per chunk), give a subagent the extraction prompt below. When it returns, paste its JSON response and write it to that chunk's file so Step B3 can collect it: + +```bash +# After pasting a subagent's JSON for chunk N, save it (replace N and PASTED_JSON): +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +cat > "${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" <<'CHUNK_JSON' +PASTED_JSON +CHUNK_JSON +``` + +Repeat for every chunk. Each chunk's JSON must land in its own `graphify-out/.graphify_chunk_NN.json` before Step B3 runs. + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted, and write its response to that chunk's file. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi +``` + +## For --update and --cluster-only + +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: + +```bash +graphify query "" +``` + +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native CLAUDE.md integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md new file mode 100644 index 00000000..a15480dc --- /dev/null +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -0,0 +1,650 @@ +--- +name: graphify-windows +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + +Follow these steps in order. Do not skip steps. + +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + +### Step 1 - Ensure graphify is installed + +```powershell +# Detect Python with graphify — uv/pipx-aware (fixes #831) +New-Item -ItemType Directory -Force -Path graphify-out | Out-Null +$GRAPHIFY_PYTHON = $null + +function Find-GraphifyPython { + # 1. uv tool install — 'uv tool dir' is authoritative, respects UV_TOOL_DIR automatically + if (Get-Command uv -ErrorAction SilentlyContinue) { + $uvDir = (uv tool dir 2>$null).Trim() + if ($uvDir) { + $py = Join-Path $uvDir "graphifyy\Scripts\python.exe" + if (Test-Path $py) { + & $py -c "import graphify" 2>$null + if ($LASTEXITCODE -eq 0) { return $py } + } + } + } + # 2. pipx install — 'pipx environment' respects PIPX_HOME automatically + if (Get-Command pipx -ErrorAction SilentlyContinue) { + $venvs = (pipx environment --value PIPX_LOCAL_VENVS 2>$null).Trim() + if ($venvs) { + $py = Join-Path $venvs "graphifyy\Scripts\python.exe" + if (Test-Path $py) { + & $py -c "import graphify" 2>$null + if ($LASTEXITCODE -eq 0) { return $py } + } + } + } + # 3. Active venv / conda / pip-into-current-env + $pyCmd = Get-Command python -ErrorAction SilentlyContinue + if ($pyCmd) { + & $pyCmd.Source -c "import graphify" 2>$null + if ($LASTEXITCODE -eq 0) { + return (& $pyCmd.Source -c "import sys; print(sys.executable)").Trim() + } + } + return $null +} + +# Try to find the right Python (uv → pipx → active env) +$GRAPHIFY_PYTHON = Find-GraphifyPython + +# Not found — install then re-detect +if (-not $GRAPHIFY_PYTHON) { + if (Get-Command uv -ErrorAction SilentlyContinue) { + uv tool install --upgrade graphifyy -q 2>&1 | Select-Object -Last 3 + } else { + pip install graphifyy -q 2>&1 | Select-Object -Last 3 + } + $GRAPHIFY_PYTHON = Find-GraphifyPython +} + +# Save interpreter path — all subsequent steps read this +$GRAPHIFY_PYTHON | Out-File -FilePath graphify-out\.graphify_python -Encoding utf8 -NoNewline +# Save scan root so `graphify update` (no args) knows where to look next time +(Resolve-Path INPUT_PATH).Path | Out-File -FilePath graphify-out\.graphify_root -Encoding utf8 -NoNewline +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent block, run Python through the saved interpreter — `& (Get-Content graphify-out\.graphify_python)` in place of a bare `python3` — so every step uses the interpreter that actually has graphify.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Video and audio (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch ALL subagents in a single message** + +Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. + +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. + +Concrete example for 3 chunks: +``` +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] +``` +All three in one message. Not three separate messages. + +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```powershell +$PROJECT_ROOT = Get-Content graphify-out\.graphify_root +# Then for chunk N: $CHUNK_PATH = Join-Path $PROJECT_ROOT "graphify-out\.graphify_chunk_0N.json" +``` + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi +``` + +## For --update and --cluster-only + +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: + +```bash +graphify query "" +``` + +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native CLAUDE.md integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. + +--- + +## Troubleshooting + +### PowerShell 5.1: Vertical scrolling stops working + +If vertical scrolling breaks in PowerShell after running graphify, this is caused by ANSI escape sequences from the `graspologic` library. Graphify v0.3.10+ suppresses this output, but if you still see the issue: + +1. **Upgrade graphify**: `pip install --upgrade graphifyy` +2. **Use Windows Terminal** instead of the legacy PowerShell console — Windows Terminal handles ANSI codes correctly +3. **Reset your terminal**: close and reopen PowerShell +4. **Skip graspologic**: uninstall it (`pip uninstall graspologic`) and graphify will fall back to NetworkX's built-in Louvain algorithm, which produces no ANSI output + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md new file mode 100644 index 00000000..4e9827b1 --- /dev/null +++ b/tools/skillgen/expected/graphify__skill.md @@ -0,0 +1,615 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + +Follow these steps in order. Do not skip steps. + +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Video and audio (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch ALL subagents in a single message** + +Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. + +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. + +Concrete example for 3 chunks: +``` +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] +``` +All three in one message. Not three separate messages. + +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi +``` + +## For --update and --cluster-only + +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, run the query directly: + +```bash +graphify query "" +``` + +Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. For that vocab-expansion step, the `--dfs` / `--budget` modes, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native CLAUDE.md integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__add-watch.md b/tools/skillgen/expected/graphify__skills__amp__references__add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__amp__references__add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__exports.md b/tools/skillgen/expected/graphify__skills__amp__references__exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__amp__references__exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/tools/skillgen/expected/graphify__skills__amp__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__amp__references__github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__amp__references__github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__hooks.md b/tools/skillgen/expected/graphify__skills__amp__references__hooks.md new file mode 100644 index 00000000..af1ac7e7 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__amp__references__hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native AGENTS.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's AGENTS.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native AGENTS.md integration + +Run once per project to make graphify always-on in Amp sessions: + +```bash +graphify amp install +``` + +This writes a `## graphify` section to the local `AGENTS.md` that instructs Amp to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify amp uninstall # remove the section +``` diff --git a/tools/skillgen/expected/graphify__skills__amp__references__query.md b/tools/skillgen/expected/graphify__skills__amp__references__query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__amp__references__query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/tools/skillgen/expected/graphify__skills__amp__references__transcribe.md b/tools/skillgen/expected/graphify__skills__amp__references__transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__amp__references__transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__update.md b/tools/skillgen/expected/graphify__skills__amp__references__update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__amp__references__update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/tools/skillgen/expected/graphify__skills__claude__references__add-watch.md b/tools/skillgen/expected/graphify__skills__claude__references__add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claude__references__add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/tools/skillgen/expected/graphify__skills__claude__references__exports.md b/tools/skillgen/expected/graphify__skills__claude__references__exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claude__references__exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/tools/skillgen/expected/graphify__skills__claude__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__claude__references__github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claude__references__github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/tools/skillgen/expected/graphify__skills__claude__references__hooks.md b/tools/skillgen/expected/graphify__skills__claude__references__hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claude__references__hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/tools/skillgen/expected/graphify__skills__claude__references__query.md b/tools/skillgen/expected/graphify__skills__claude__references__query.md new file mode 100644 index 00000000..b08289e1 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claude__references__query.md @@ -0,0 +1,103 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +### Step 0 — Constrained query expansion (REQUIRED before traversal) + +graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. + +Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: + +1. Extract the token vocabulary from node labels: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, re +from pathlib import Path +data = json.loads(Path('graphify-out/graph.json').read_text()) +vocab = set() +for n in data['nodes']: + for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): + parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] + for p in parts: + t = p.lower() + if 3 <= len(t) <= 30: + vocab.add(t) +Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab))) +print(f'vocab: {len(vocab)} tokens') +" +``` + +2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: + - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. + - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. + - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. + - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. + - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. + +3. Print the selection explicitly to the user before running the query, so the expansion is auditable: +``` +Query expanded to (from graph vocab, N tokens): [token1, token2, ...] +``` +If the list is empty, say so plainly and stop — do not proceed to traversal. + +### Step 1 — Traversal + +Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) + +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +Answer using **only** what the graph output contains. Quote `source_location` when citing a specific fact. If the graph lacks enough information, say so - do not hallucinate edges. + +After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +graphify path "NODE_A" "NODE_B" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +graphify explain "NODE_NAME" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/tools/skillgen/expected/graphify__skills__claude__references__transcribe.md b/tools/skillgen/expected/graphify__skills__claude__references__transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claude__references__transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__claude__references__update.md b/tools/skillgen/expected/graphify__skills__claude__references__update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claude__references__update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/tools/skillgen/expected/graphify__skills__claw__references__add-watch.md b/tools/skillgen/expected/graphify__skills__claw__references__add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claw__references__add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/tools/skillgen/expected/graphify__skills__claw__references__exports.md b/tools/skillgen/expected/graphify__skills__claw__references__exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claw__references__exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md new file mode 100644 index 00000000..231da82b --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md @@ -0,0 +1,29 @@ +# graphify reference: extraction subagent prompt (compact) + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation) +- INFERRED: reasonable inference (shared structure, implied dependency) +- AMBIGUOUS: uncertain — flag it, do not omit +- Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. +- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Image files: use vision — understand what the image IS, not just OCR +- DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. +- Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. +- Hyperedges: if 3+ nodes share a concept, flow, or pattern not captured by pairwise edges, add a hyperedge to a top-level `hyperedges` array. Use sparingly. Max 3 per chunk. +- If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. +- confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. + +Output exactly this JSON (no other text): +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +``` diff --git a/tools/skillgen/expected/graphify__skills__claw__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__claw__references__github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claw__references__github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/tools/skillgen/expected/graphify__skills__claw__references__hooks.md b/tools/skillgen/expected/graphify__skills__claw__references__hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claw__references__hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/tools/skillgen/expected/graphify__skills__claw__references__query.md b/tools/skillgen/expected/graphify__skills__claw__references__query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claw__references__query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/tools/skillgen/expected/graphify__skills__claw__references__transcribe.md b/tools/skillgen/expected/graphify__skills__claw__references__transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claw__references__transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__claw__references__update.md b/tools/skillgen/expected/graphify__skills__claw__references__update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claw__references__update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/tools/skillgen/expected/graphify__skills__codex__references__add-watch.md b/tools/skillgen/expected/graphify__skills__codex__references__add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__codex__references__add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/tools/skillgen/expected/graphify__skills__codex__references__exports.md b/tools/skillgen/expected/graphify__skills__codex__references__exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__codex__references__exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md new file mode 100644 index 00000000..231da82b --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md @@ -0,0 +1,29 @@ +# graphify reference: extraction subagent prompt (compact) + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation) +- INFERRED: reasonable inference (shared structure, implied dependency) +- AMBIGUOUS: uncertain — flag it, do not omit +- Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. +- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Image files: use vision — understand what the image IS, not just OCR +- DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. +- Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. +- Hyperedges: if 3+ nodes share a concept, flow, or pattern not captured by pairwise edges, add a hyperedge to a top-level `hyperedges` array. Use sparingly. Max 3 per chunk. +- If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. +- confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. + +Output exactly this JSON (no other text): +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +``` diff --git a/tools/skillgen/expected/graphify__skills__codex__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__codex__references__github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__codex__references__github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/tools/skillgen/expected/graphify__skills__codex__references__hooks.md b/tools/skillgen/expected/graphify__skills__codex__references__hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__codex__references__hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/tools/skillgen/expected/graphify__skills__codex__references__query.md b/tools/skillgen/expected/graphify__skills__codex__references__query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__codex__references__query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/tools/skillgen/expected/graphify__skills__codex__references__transcribe.md b/tools/skillgen/expected/graphify__skills__codex__references__transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__codex__references__transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__codex__references__update.md b/tools/skillgen/expected/graphify__skills__codex__references__update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__codex__references__update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__add-watch.md b/tools/skillgen/expected/graphify__skills__copilot__references__add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__copilot__references__add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__exports.md b/tools/skillgen/expected/graphify__skills__copilot__references__exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__copilot__references__exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__copilot__references__github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__copilot__references__github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__hooks.md b/tools/skillgen/expected/graphify__skills__copilot__references__hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__copilot__references__hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__query.md b/tools/skillgen/expected/graphify__skills__copilot__references__query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__copilot__references__query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__transcribe.md b/tools/skillgen/expected/graphify__skills__copilot__references__transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__copilot__references__transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__update.md b/tools/skillgen/expected/graphify__skills__copilot__references__update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__copilot__references__update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/tools/skillgen/expected/graphify__skills__droid__references__add-watch.md b/tools/skillgen/expected/graphify__skills__droid__references__add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__droid__references__add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/tools/skillgen/expected/graphify__skills__droid__references__exports.md b/tools/skillgen/expected/graphify__skills__droid__references__exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__droid__references__exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/tools/skillgen/expected/graphify__skills__droid__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__droid__references__github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__droid__references__github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/tools/skillgen/expected/graphify__skills__droid__references__hooks.md b/tools/skillgen/expected/graphify__skills__droid__references__hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__droid__references__hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/tools/skillgen/expected/graphify__skills__droid__references__query.md b/tools/skillgen/expected/graphify__skills__droid__references__query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__droid__references__query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/tools/skillgen/expected/graphify__skills__droid__references__transcribe.md b/tools/skillgen/expected/graphify__skills__droid__references__transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__droid__references__transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__droid__references__update.md b/tools/skillgen/expected/graphify__skills__droid__references__update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__droid__references__update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__add-watch.md b/tools/skillgen/expected/graphify__skills__kilo__references__add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kilo__references__add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__exports.md b/tools/skillgen/expected/graphify__skills__kilo__references__exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kilo__references__exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__kilo__references__github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kilo__references__github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__hooks.md b/tools/skillgen/expected/graphify__skills__kilo__references__hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kilo__references__hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__query.md b/tools/skillgen/expected/graphify__skills__kilo__references__query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kilo__references__query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__transcribe.md b/tools/skillgen/expected/graphify__skills__kilo__references__transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kilo__references__transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__update.md b/tools/skillgen/expected/graphify__skills__kilo__references__update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kilo__references__update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__add-watch.md b/tools/skillgen/expected/graphify__skills__kiro__references__add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kiro__references__add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__exports.md b/tools/skillgen/expected/graphify__skills__kiro__references__exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kiro__references__exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md new file mode 100644 index 00000000..231da82b --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md @@ -0,0 +1,29 @@ +# graphify reference: extraction subagent prompt (compact) + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation) +- INFERRED: reasonable inference (shared structure, implied dependency) +- AMBIGUOUS: uncertain — flag it, do not omit +- Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. +- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Image files: use vision — understand what the image IS, not just OCR +- DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. +- Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. +- Hyperedges: if 3+ nodes share a concept, flow, or pattern not captured by pairwise edges, add a hyperedge to a top-level `hyperedges` array. Use sparingly. Max 3 per chunk. +- If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. +- confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. + +Output exactly this JSON (no other text): +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +``` diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__kiro__references__github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kiro__references__github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__hooks.md b/tools/skillgen/expected/graphify__skills__kiro__references__hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kiro__references__hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__query.md b/tools/skillgen/expected/graphify__skills__kiro__references__query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kiro__references__query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__transcribe.md b/tools/skillgen/expected/graphify__skills__kiro__references__transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kiro__references__transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__update.md b/tools/skillgen/expected/graphify__skills__kiro__references__update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kiro__references__update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__add-watch.md b/tools/skillgen/expected/graphify__skills__opencode__references__add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__opencode__references__add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__exports.md b/tools/skillgen/expected/graphify__skills__opencode__references__exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__opencode__references__exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__opencode__references__github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__opencode__references__github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__hooks.md b/tools/skillgen/expected/graphify__skills__opencode__references__hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__opencode__references__hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__query.md b/tools/skillgen/expected/graphify__skills__opencode__references__query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__opencode__references__query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__transcribe.md b/tools/skillgen/expected/graphify__skills__opencode__references__transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__opencode__references__transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__update.md b/tools/skillgen/expected/graphify__skills__opencode__references__update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__opencode__references__update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/tools/skillgen/expected/graphify__skills__pi__references__add-watch.md b/tools/skillgen/expected/graphify__skills__pi__references__add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__pi__references__add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/tools/skillgen/expected/graphify__skills__pi__references__exports.md b/tools/skillgen/expected/graphify__skills__pi__references__exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__pi__references__exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md new file mode 100644 index 00000000..231da82b --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md @@ -0,0 +1,29 @@ +# graphify reference: extraction subagent prompt (compact) + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation) +- INFERRED: reasonable inference (shared structure, implied dependency) +- AMBIGUOUS: uncertain — flag it, do not omit +- Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. +- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Image files: use vision — understand what the image IS, not just OCR +- DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. +- Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. +- Hyperedges: if 3+ nodes share a concept, flow, or pattern not captured by pairwise edges, add a hyperedge to a top-level `hyperedges` array. Use sparingly. Max 3 per chunk. +- If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. +- confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. + +Output exactly this JSON (no other text): +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +``` diff --git a/tools/skillgen/expected/graphify__skills__pi__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__pi__references__github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__pi__references__github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/tools/skillgen/expected/graphify__skills__pi__references__hooks.md b/tools/skillgen/expected/graphify__skills__pi__references__hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__pi__references__hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/tools/skillgen/expected/graphify__skills__pi__references__query.md b/tools/skillgen/expected/graphify__skills__pi__references__query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__pi__references__query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/tools/skillgen/expected/graphify__skills__pi__references__transcribe.md b/tools/skillgen/expected/graphify__skills__pi__references__transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__pi__references__transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__pi__references__update.md b/tools/skillgen/expected/graphify__skills__pi__references__update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__pi__references__update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/tools/skillgen/expected/graphify__skills__trae__references__add-watch.md b/tools/skillgen/expected/graphify__skills__trae__references__add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__trae__references__add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/tools/skillgen/expected/graphify__skills__trae__references__exports.md b/tools/skillgen/expected/graphify__skills__trae__references__exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__trae__references__exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/tools/skillgen/expected/graphify__skills__trae__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__trae__references__github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__trae__references__github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/tools/skillgen/expected/graphify__skills__trae__references__hooks.md b/tools/skillgen/expected/graphify__skills__trae__references__hooks.md new file mode 100644 index 00000000..7c04d5b0 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__trae__references__hooks.md @@ -0,0 +1,35 @@ +# graphify reference: commit hook and native AGENTS.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's AGENTS.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native AGENTS.md integration (Trae) + +Run once per project to make graphify always-on in Trae sessions: + +```bash +graphify trae install # or: graphify trae-cn install +``` + +This writes a `## graphify` section to the local `AGENTS.md` that instructs Trae to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +> **Note:** Unlike Claude Code, Trae does NOT support PreToolUse hooks. The AGENTS.md rules are the always-on mechanism — there is no automatic graph rebuild on tool use. Run `/graphify --update` manually after code changes if the graph needs refreshing. + +```bash +graphify trae uninstall # or: graphify trae-cn uninstall # remove the section +``` diff --git a/tools/skillgen/expected/graphify__skills__trae__references__query.md b/tools/skillgen/expected/graphify__skills__trae__references__query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__trae__references__query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/tools/skillgen/expected/graphify__skills__trae__references__transcribe.md b/tools/skillgen/expected/graphify__skills__trae__references__transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__trae__references__transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__trae__references__update.md b/tools/skillgen/expected/graphify__skills__trae__references__update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__trae__references__update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__add-watch.md b/tools/skillgen/expected/graphify__skills__vscode__references__add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__vscode__references__add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__exports.md b/tools/skillgen/expected/graphify__skills__vscode__references__exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__vscode__references__exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__vscode__references__github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__vscode__references__github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__hooks.md b/tools/skillgen/expected/graphify__skills__vscode__references__hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__vscode__references__hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__query.md b/tools/skillgen/expected/graphify__skills__vscode__references__query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__vscode__references__query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__transcribe.md b/tools/skillgen/expected/graphify__skills__vscode__references__transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__vscode__references__transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__update.md b/tools/skillgen/expected/graphify__skills__vscode__references__update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__vscode__references__update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/tools/skillgen/expected/graphify__skills__windows__references__add-watch.md b/tools/skillgen/expected/graphify__skills__windows__references__add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__windows__references__add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/tools/skillgen/expected/graphify__skills__windows__references__exports.md b/tools/skillgen/expected/graphify__skills__windows__references__exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__windows__references__exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/tools/skillgen/expected/graphify__skills__windows__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__windows__references__github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__windows__references__github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/tools/skillgen/expected/graphify__skills__windows__references__hooks.md b/tools/skillgen/expected/graphify__skills__windows__references__hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__windows__references__hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/tools/skillgen/expected/graphify__skills__windows__references__query.md b/tools/skillgen/expected/graphify__skills__windows__references__query.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__windows__references__query.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/tools/skillgen/expected/graphify__skills__windows__references__transcribe.md b/tools/skillgen/expected/graphify__skills__windows__references__transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__windows__references__transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/tools/skillgen/expected/graphify__skills__windows__references__update.md b/tools/skillgen/expected/graphify__skills__windows__references__update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__windows__references__update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/tools/skillgen/fragments/always-on/agents-md.md b/tools/skillgen/fragments/always-on/agents-md.md new file mode 100644 index 00000000..20cff728 --- /dev/null +++ b/tools/skillgen/fragments/always-on/agents-md.md @@ -0,0 +1,12 @@ +## graphify + +This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. + +When the user types `/graphify`, invoke the `skill` tool with `skill: "graphify"` before doing anything else. + +Rules: +- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- Dirty graphify-out/ files are expected after hooks or incremental updates; dirty graph files are not a reason to skip graphify. Only skip graphify if the task is about stale or incorrect graph output, or the user explicitly says not to use it. +- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. +- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/tools/skillgen/fragments/always-on/antigravity-rules.md b/tools/skillgen/fragments/always-on/antigravity-rules.md new file mode 100644 index 00000000..0fc78641 --- /dev/null +++ b/tools/skillgen/fragments/always-on/antigravity-rules.md @@ -0,0 +1,14 @@ +--- +trigger: always_on +description: Consult the graphify knowledge graph at graphify-out/ for codebase and architecture questions. +--- + +## graphify + +This project has a graphify knowledge graph at graphify-out/. + +Rules: +- For codebase or architecture questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (CLI) or `query_graph` (MCP). Use `graphify path "" ""` / `shortest_path` for relationships and `graphify explain ""` / `get_node` for focused concepts. These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. +- If graphify-out/wiki/index.md exists, navigate it instead of reading raw files +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context +- After modifying code files in this session, run `graphify update .` to keep the graph current (AST-only, no API cost) diff --git a/tools/skillgen/fragments/always-on/claude-md.md b/tools/skillgen/fragments/always-on/claude-md.md new file mode 100644 index 00000000..417efeb2 --- /dev/null +++ b/tools/skillgen/fragments/always-on/claude-md.md @@ -0,0 +1,9 @@ +## graphify + +This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. + +Rules: +- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. +- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/tools/skillgen/fragments/always-on/gemini-md.md b/tools/skillgen/fragments/always-on/gemini-md.md new file mode 100644 index 00000000..417efeb2 --- /dev/null +++ b/tools/skillgen/fragments/always-on/gemini-md.md @@ -0,0 +1,9 @@ +## graphify + +This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. + +Rules: +- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. +- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/tools/skillgen/fragments/always-on/kiro-steering.md b/tools/skillgen/fragments/always-on/kiro-steering.md new file mode 100644 index 00000000..cb6f4543 --- /dev/null +++ b/tools/skillgen/fragments/always-on/kiro-steering.md @@ -0,0 +1,5 @@ +--- +inclusion: always +--- + +graphify: A knowledge graph of this project lives in `graphify-out/`. For codebase, architecture, or dependency questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. Read `GRAPH_REPORT.md` only for broad architecture review or when those commands do not surface enough context. diff --git a/tools/skillgen/fragments/always-on/vscode-instructions.md b/tools/skillgen/fragments/always-on/vscode-instructions.md new file mode 100644 index 00000000..9cb983c9 --- /dev/null +++ b/tools/skillgen/fragments/always-on/vscode-instructions.md @@ -0,0 +1,17 @@ +## graphify + +For any question about this repo's architecture, structure, components, or how to add/modify/find +code, your first action should be `graphify query ""` when `graphify-out/graph.json` +exists. Use `graphify path "" ""` for relationship questions and `graphify explain ""` +for focused-concept questions. These return a scoped subgraph, usually much smaller than the full +report or raw grep output. + +Triggers: "how do I…", "where is…", "what does … do", "add/modify a ", +"explain the architecture", or anything that depends on how files or classes relate. + +If `graphify-out/wiki/index.md` exists, use it for broad navigation. Read `graphify-out/GRAPH_REPORT.md` +only for broad architecture review or when query/path/explain do not surface enough context. Only read +source files when (a) modifying/debugging specific code, (b) the graph lacks the needed detail, or +(c) the graph is missing or stale. + +Type `/graphify` in Copilot Chat to build or update the graph. diff --git a/tools/skillgen/fragments/core/aider.md b/tools/skillgen/fragments/core/aider.md new file mode 100644 index 00000000..aad4ff95 --- /dev/null +++ b/tools/skillgen/fragments/core/aider.md @@ -0,0 +1,1246 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. + +Three things it does that your AI assistant alone cannot: +1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. +2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. +3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. + +Use it for: +- A codebase you're new to (understand architecture before touching anything) +- A reading list (papers + tweets + notes → one navigable graph) +- A research corpus (citation graph + concept graph in one) +- Your personal /raw folder (drop everything in, let it grow, query it) + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +Follow these steps in order. Do not skip steps. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result)) +" > .graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. + +**Step 2 - Transcribe:** + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('.graphify_detect.json').read_text()) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files) + Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +> **Aider platform:** Multi-agent support is still early on Aider. Extraction runs sequentially — you read and extract each file yourself. This is slower than parallel platforms but fully reliable. + +Print: `"Semantic extraction: N files (sequential — Aider)"` + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('.graphify_detect.json').read_text()) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) +Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Sequential extraction (Aider)** + +Process each file one at a time. For each file: + +1. Read the file contents +2. Extract nodes, edges, and hyperedges applying the same rules: + - EXTRACTED: relationship explicit in source (import, call, citation) + - INFERRED: reasonable inference (shared structure, implied dependency) + - AMBIGUOUS: uncertain — flag it, do not omit + - Code files: semantic edges AST cannot find. Do not re-extract imports. + - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. When adding `calls` edges: source is caller, target is callee. + - Image files: use vision — understand what the image IS, not just OCR + - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges + - Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. + - Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. + - confidence_score REQUIRED on every edge: EXTRACTED=1.0, INFERRED=0.6-0.9 (reason individually), AMBIGUOUS=0.1-0.3 +3. Accumulate results across all files + +Schema for each file's output: +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +After processing all files, write the accumulated result to `.graphify_semantic_new.json`. + +**Step B3 - Cache and merge** + +For the accumulated result: + +If more than half the chunks failed, stop and tell the user. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text()) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2)) +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('.graphify_ast.json').read_text()) +sem = json.loads(Path('.graphify_semantic.json').read_text()) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +detection = json.loads(Path('.graphify_detect.json').read_text()) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +detection = json.loads(Path('.graphify_detect.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_obsidian, to_canvas +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) +print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') + +to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) +print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') +print() +print('Open graphify-out/obsidian/ as a vault in Obsidian.') +print(' Graph view - nodes colored by community (set automatically)') +print(' graph.canvas - structured layout with communities as groups') +print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') +" +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_html +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +if G.number_of_nodes() > 5000: + print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') +else: + to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) + print('graph.html written - open in any browser, no server needed') +" +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_cypher +from pathlib import Path + +G = build_from_json(json.loads(Path('.graphify_extract.json').read_text())) +to_cypher(G, 'graphify-out/cypher.txt') +print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') +" +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster +from graphify.export import push_to_neo4j +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') +" +``` + +Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_svg +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) +print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') +" +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.build import build_from_json +from graphify.export import to_graphml +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +to_graphml(G, communities, 'graphify-out/graph.graphml') +print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') +" +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `.graphify_detect.json` is greater than 5,000, run: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.benchmark import run_benchmark, print_benchmark +from pathlib import Path + +detection = json.loads(Path('.graphify_detect.json').read_text()) +result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) +print_benchmark(result) +" +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('.graphify_detect.json').read_text()) +save_manifest(detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('.graphify_extract.json').read_text()) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text()) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2)) + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2)) +Path('.graphify_incremental.json').write_text(json.dumps(result)) +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load existing graph +existing_data = json.loads(Path('graphify-out/graph.json').read_text()) +G_existing = json_graph.node_link_graph(existing_data, edges='links') + +# Load new extraction +new_extraction = json.loads(Path('.graphify_extract.json').read_text()) +G_new = build_from_json(new_extraction) + +# Merge: new nodes/edges into existing graph +G_existing.update(G_new) +print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None +new_extract = json.loads(Path('.graphify_extract.json').read_text()) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` +Clean up after: `rm -f .graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections +from graphify.report import generate +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, + 'files': {'code': [], 'document': [], 'paper': []}} +tokens = {'input': 0, 'output': 0} + +communities = cluster(G) +cohesion = score_all(G, communities) +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, +} +Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +print(f'Re-clustered: {len(communities)} communities') +" +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). + +--- + +## For /graphify query + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Load `graphify-out/graph.json`, then: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` + +--- + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. + +--- + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md new file mode 100644 index 00000000..4a56635e --- /dev/null +++ b/tools/skillgen/fragments/core/core.md @@ -0,0 +1,543 @@ +@@FRONTMATTER@@ + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. + +Follow these steps in order. Do not skip steps. + +### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) + +Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. + +### Step 1 - Ensure graphify is installed + +@@INSTALL@@ + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result, ensure_ascii=False)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: + - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). + - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. + - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. + - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. + - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Video and audio (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: +> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). + +Print it once, then continue. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching Claude subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. + +> **No other API keys are read.** If `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, fall straight through to Claude Code subagent dispatch (Part B below) — the host session itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key from the environment. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files, cache_root=Path('.')) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +@@DISPATCH@@ + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text(encoding=\"utf-8\")) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, '.', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. + +```bash +graphify export obsidian +# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +graphify export html # auto-aggregates to community view if graph > 5000 nodes +# or: graphify export html --no-viz +``` + +### Steps 6b-8 - Wiki, Neo4j, SVG, GraphML, MCP, benchmark (only on their flags) + +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed +# subset. Full-rebuild mode populates only 'files', so the fallback handles that. +save_manifest(detect.get('all_files') or detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +fi +``` + +## For --update and --cluster-only + +Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. + +--- + +## For /graphify query + +@@QUERY_STUB@@ + +--- + +## For /graphify add and --watch + +Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. + +--- + +## For the commit hook and native @@HOOKS_TARGET@@ integration + +When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's @@HOOKS_TARGET@@, see `references/hooks.md`. + +--- + +@@EXTRA@@## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/tools/skillgen/fragments/core/devin.md b/tools/skillgen/fragments/core/devin.md new file mode 100644 index 00000000..7013159a --- /dev/null +++ b/tools/skillgen/fragments/core/devin.md @@ -0,0 +1,1372 @@ +--- +name: graphify +description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +argument-hint: "[path|query|subcommand]" +model: sonnet +allowed-tools: + - read + - grep + - glob + - exec +triggers: + - user + - model +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory +/graphify # full pipeline on specific path +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. + +Three things it does that your AI assistant alone cannot: +1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. +2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. +3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. + +Use it for: +- A codebase you're new to (understand architecture before touching anything) +- A reading list (papers + tweets + notes -> one navigable graph) +- A research corpus (citation graph + concept graph in one) +- Your personal /raw folder (drop everything in, let it grow, query it) + +## What You Must Do When Invoked + +If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +Follow these steps in order. Do not skip steps. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Force UTF-8 I/O on Windows (prevents garbled CJK/non-ASCII output) +export PYTHONUTF8=1 +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. + +**Step 2 - Transcribe:** + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2)) + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**MANDATORY: You MUST use the subagent system here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use parallel subagents you are doing this wrong.** + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached)) +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch subagents** + +Dispatch ALL subagents in the same response so they run in parallel. + +Concrete example for 3 chunks: +``` +[Subagent 1: files 1-15] +[Subagent 2: files 16-30] +[Subagent 3: files 31-45] +``` +All three in one message. Not three separate messages. + +For each chunk, dispatch a subagent with this exact prompt (fill in FILE_LIST): + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON with no commentary: {"nodes": [...], "edges": [...], "hyperedges": [...], "input_tokens": 0, "output_tokens": 0} + +Extraction rules: +- EXTRACTED: relationship explicit in source (import, call, citation) +- INFERRED: reasonable inference (shared structure, implied dependency) +- AMBIGUOUS: uncertain — flag it, do not omit +- Code files: extract semantic edges AST cannot find (design patterns, protocol conformance). Do not re-extract imports. +- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. When adding `calls` edges: source is caller, target is callee. +- Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. +- DEEP_MODE (if set): be aggressive with INFERRED edges +- Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. +- Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. +- If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. +- confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Schema: +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Files: +FILE_LIST +``` + +**Step B3 - Cache and merge** + +Wait for all subagents. For each result: +- Check that `graphify-out/.graphify_chunk_N.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. + +After each subagent call completes, write its result to `graphify-out/.graphify_chunk_N.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then merge: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path +from graphify.semantic_cleanup import load_validated_semantic_fragment, sanitize_semantic_fragment + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d, errors = load_validated_semantic_fragment(Path(c)) + if errors: + print(f'Skipping invalid chunk {c}: ' + '; '.join(errors[:3])) + continue + d = sanitize_semantic_fragment(d) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2)) +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.semantic_cleanup import sanitize_semantic_fragment + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text()) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +merged = sanitize_semantic_fragment(merged) +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path +from graphify.semantic_cleanup import sanitize_semantic_fragment + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text()) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text()) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +merged = sanitize_semantic_fragment(merged) +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_obsidian, to_canvas +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) +print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') + +to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) +print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') +print() +print('Open graphify-out/obsidian/ as a vault in Obsidian.') +print(' Graph view - nodes colored by community (set automatically)') +print(' graph.canvas - structured layout with communities as groups') +print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') +" +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_html +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +NODE_LIMIT = 5000 +if G.number_of_nodes() > NODE_LIMIT: + from collections import Counter + print(f'Graph has {G.number_of_nodes()} nodes (above {NODE_LIMIT} limit). Building aggregated community view...') + node_to_community = {nid: cid for cid, members in communities.items() for nid in members} + import networkx as nx_meta + meta = nx_meta.Graph() + for cid, members in communities.items(): + meta.add_node(str(cid), label=labels.get(cid, f'Community {cid}')) + edge_counts = Counter() + for u, v in G.edges(): + cu, cv = node_to_community.get(u), node_to_community.get(v) + if cu is not None and cv is not None and cu != cv: + edge_counts[(min(cu, cv), max(cu, cv))] += 1 + for (cu, cv), w in edge_counts.items(): + meta.add_edge(str(cu), str(cv), weight=w, relation=f'{w} cross-community edges', confidence='AGGREGATED') + if meta.number_of_nodes() > 1: + meta_communities = {cid: [str(cid)] for cid in communities} + member_counts = {cid: len(members) for cid, members in communities.items()} + to_html(meta, meta_communities, 'graphify-out/graph.html', community_labels=labels or None, member_counts=member_counts) + print(f'graph.html written (aggregated: {meta.number_of_nodes()} community nodes, {meta.number_of_edges()} cross-community edges)') + print('Tip: run with --obsidian for full node-level detail, or --wiki for an agent-crawlable wiki.') + else: + print('Single community — aggregated view not useful. Skipping graph.html.') +else: + to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) + print('graph.html written - open in any browser, no server needed') +" +``` + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +The wiki is an agent-crawlable export — `index.md` plus one article per community plus god-node articles. It is the recommended fallback for graphs too large to render as HTML, and it's the most useful output for an autonomous agent navigating the graph between sessions. + +Run this before Step 9 (cleanup) so `graphify-out/.graphify_labels.json` is still available. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.build import build_from_json +from graphify.wiki import to_wiki +from graphify.analyze import god_nodes +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +labels = {int(k): v for k, v in labels_raw.items()} +gods = god_nodes(G) + +n = to_wiki(G, communities, 'graphify-out/wiki', community_labels=labels or None, cohesion=cohesion, god_nodes_data=gods) +print(f'Wiki: {n} articles written to graphify-out/wiki/') +print(' graphify-out/wiki/index.md -> agent entry point') +" +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_cypher +from pathlib import Path + +G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text())) +to_cypher(G, 'graphify-out/cypher.txt') +print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') +" +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import push_to_neo4j +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') +" +``` + +Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_svg +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) +print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') +" +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.build import build_from_json +from graphify.export import to_graphml +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +to_graphml(G, communities, 'graphify-out/graph.graphml') +print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') +" +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.benchmark import run_benchmark, print_benchmark +from pathlib import Path + +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) +print_benchmark(result) +" +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +save_manifest(detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text()) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2)) + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json graphify-out/.graphify_chunk_*.json graphify-out/.graphify_incremental.json graphify-out/.graphify_transcripts.json graphify-out/.graphify_old.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given; omit the wiki line unless --wiki was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) + wiki/ - agent-crawlable wiki, start at wiki/index.md (only if --wiki was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" +fi +``` + +--- + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result)) +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4-8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A-3C pipeline as normal. + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load existing graph +existing_data = json.loads(Path('graphify-out/graph.json').read_text()) +G_existing = json_graph.node_link_graph(existing_data, edges='links') + +# Load new extraction +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +G_new = build_from_json(new_extraction) + +# Merge: new nodes/edges into existing graph +G_existing.update(G_new) +print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') +" +``` + +Then run Steps 4-8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text()) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1-3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections +from graphify.report import generate +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, + 'files': {'code': [], 'document': [], 'paper': []}} +tokens = {'input': 0, 'output': 0} + +communities = cluster(G) +cohesion = score_all(G, communities) +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +print(f'Re-clustered: {len(communities)} communities') +" +``` + +Then run Steps 5-9 as normal (label communities, generate viz, benchmark, clean up, report). + +--- + +## For /graphify query + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Load `graphify-out/graph.json`, then: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept. Then write a 3-5 sentence explanation using source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` + +--- + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- Twitter/X -> fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv -> abstract + metadata saved as `.md` +- PDF -> downloaded as `.pdf` +- Images (.png/.jpg/.webp) -> downloaded, vision extraction runs on next build +- Any webpage -> converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +--- + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed, re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +--- + +## For always-on context in Devin sessions + +Run once per project to make graphify always-on in Devin sessions: + +```bash +graphify devin install --project +``` + +This writes a `## graphify` section to `.windsurf/rules/graphify.md` that instructs Devin to check the graph before answering codebase questions and rebuild it after code changes. + +```bash +graphify devin uninstall --project # remove +``` + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. diff --git a/tools/skillgen/fragments/dispatch/.gitkeep b/tools/skillgen/fragments/dispatch/.gitkeep new file mode 100644 index 00000000..0276e5b8 --- /dev/null +++ b/tools/skillgen/fragments/dispatch/.gitkeep @@ -0,0 +1 @@ +# Reserved for later waves (codex/windows/remaining hosts). Empty in wave 2. diff --git a/tools/skillgen/fragments/dispatch/agent-tool-disk-powershell.md b/tools/skillgen/fragments/dispatch/agent-tool-disk-powershell.md new file mode 100644 index 00000000..8116ee82 --- /dev/null +++ b/tools/skillgen/fragments/dispatch/agent-tool-disk-powershell.md @@ -0,0 +1,25 @@ +**Step B2 - Dispatch ALL subagents in a single message** + +Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. + +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. + +Concrete example for 3 chunks: +``` +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] +``` +All three in one message. Not three separate messages. + +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```powershell +$PROJECT_ROOT = Get-Content graphify-out\.graphify_root +# Then for chunk N: $CHUNK_PATH = Join-Path $PROJECT_ROOT "graphify-out\.graphify_chunk_0N.json" +``` + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. diff --git a/tools/skillgen/fragments/dispatch/agent-tool-disk.md b/tools/skillgen/fragments/dispatch/agent-tool-disk.md new file mode 100644 index 00000000..912498cf --- /dev/null +++ b/tools/skillgen/fragments/dispatch/agent-tool-disk.md @@ -0,0 +1,25 @@ +**Step B2 - Dispatch ALL subagents in a single message** + +Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. + +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. + +Concrete example for 3 chunks: +``` +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] +``` +All three in one message. Not three separate messages. + +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. diff --git a/tools/skillgen/fragments/dispatch/codex-agenttask.md b/tools/skillgen/fragments/dispatch/codex-agenttask.md new file mode 100644 index 00000000..1fd2f6a1 --- /dev/null +++ b/tools/skillgen/fragments/dispatch/codex-agenttask.md @@ -0,0 +1,22 @@ +**Step B2 - Dispatch ALL subagents in a single message (Codex)** + +> **Codex platform:** Uses `spawn_agent` + `wait_agent` + `close_agent` instead of the Agent tool. +> Requires `multi_agent = true` under `[features]` in `~/.codex/config.toml`. +> If `spawn_agent` is unavailable, tell the user to add that config and restart Codex. + +Call `spawn_agent` once per chunk — ALL in the same response so they run in parallel. Build the message by wrapping the extraction prompt in task-delegation framing: + +``` +spawn_agent(agent_type="worker", message="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") +``` + +After all agents are dispatched, collect results sequentially in memory: +``` +result = wait_agent(handle); close_agent(handle) # repeat per handle +``` + +Parse each result as JSON. Accumulate nodes/edges/hyperedges across all results and write to `graphify-out/.graphify_semantic_new.json`. Codex collects in memory, so there are no per-chunk files on disk; the disk-based success checks in Step B3 do not apply — a chunk that returns invalid JSON is the failure signal instead. + +Subagent prompt template: + +See `references/extraction-spec.md` for the compact subagent prompt (rules, node-ID format, confidence rubric, hyperedge and vision rules, JSON schema). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each agent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted, and have it return the JSON inline. diff --git a/tools/skillgen/fragments/dispatch/manual-paste.md b/tools/skillgen/fragments/dispatch/manual-paste.md new file mode 100644 index 00000000..589855ad --- /dev/null +++ b/tools/skillgen/fragments/dispatch/manual-paste.md @@ -0,0 +1,21 @@ +**Step B2 - Dispatch subagents and paste their responses** + +> **No automated subagent tool:** this host has no parallel Agent/Task API, so +> extraction is driven by hand. Dispatch a subagent per chunk however the host +> allows (a fresh conversation, a parallel pane), then paste each response back. + +For each chunk of uncached files (20-25 per chunk), give a subagent the extraction prompt below. When it returns, paste its JSON response and write it to that chunk's file so Step B3 can collect it: + +```bash +# After pasting a subagent's JSON for chunk N, save it (replace N and PASTED_JSON): +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +cat > "${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" <<'CHUNK_JSON' +PASTED_JSON +CHUNK_JSON +``` + +Repeat for every chunk. Each chunk's JSON must land in its own `graphify-out/.graphify_chunk_NN.json` before Step B3 runs. + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted, and write its response to that chunk's file. diff --git a/tools/skillgen/fragments/dispatch/opencode-mention.md b/tools/skillgen/fragments/dispatch/opencode-mention.md new file mode 100644 index 00000000..f87f5fb5 --- /dev/null +++ b/tools/skillgen/fragments/dispatch/opencode-mention.md @@ -0,0 +1,17 @@ +**Step B2 - Dispatch ALL subagents in a single message (OpenCode)** + +> **OpenCode platform:** Uses `@mention` dispatch instead of the Agent tool. All mentions in a single message run in parallel. + +Dispatch one `@mention` per chunk — ALL in the same response: + +``` +@agent Chunk CHUNK_NUM of TOTAL_CHUNKS: [extraction prompt with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted] + +@agent Chunk 2 of TOTAL_CHUNKS: [next chunk] +``` + +Wait for all agents to return. Parse each response as JSON. Accumulate nodes/edges/hyperedges across all results and write to `graphify-out/.graphify_semantic_new.json`. If the `@agent` path cannot write chunk files, fall back to the serial path that writes each `graphify-out/.graphify_chunk_NN.json` before merge. + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each agent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted. diff --git a/tools/skillgen/fragments/dispatch/task-tool-disk-trae.md b/tools/skillgen/fragments/dispatch/task-tool-disk-trae.md new file mode 100644 index 00000000..fdd3be0e --- /dev/null +++ b/tools/skillgen/fragments/dispatch/task-tool-disk-trae.md @@ -0,0 +1,23 @@ +**Step B2 - Dispatch ALL subagents in a single message** + +> Uses the `Task` tool for parallel subagent dispatch. +> Call `Task` once per chunk — ALL in the same response so they run in parallel. +> Trae does NOT support PreToolUse hooks — AGENTS.md rules are the always-on mechanism instead. + +Pass the extraction prompt as the task description: + +``` +Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") +``` + +Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. + +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. diff --git a/tools/skillgen/fragments/dispatch/task-tool-disk.md b/tools/skillgen/fragments/dispatch/task-tool-disk.md new file mode 100644 index 00000000..91098730 --- /dev/null +++ b/tools/skillgen/fragments/dispatch/task-tool-disk.md @@ -0,0 +1,22 @@ +**Step B2 - Dispatch ALL subagents in a single message** + +> Uses the `Task` tool for parallel subagent dispatch. +> Call `Task` once per chunk — ALL in the same response so they run in parallel. + +Pass the extraction prompt as the task description: + +``` +Task(description="Your task is to perform the following. Follow the instructions below exactly.\n\n\n[extraction prompt, with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted]\n\n\nExecute this now. Output ONLY the structured JSON response.") +``` + +Each subagent writes its result to its own `graphify-out/.graphify_chunk_NN.json`. Collect results as each `Task` completes and parse each as JSON. + +CHUNK_PATH must be an **absolute** path — derive it before dispatching: +```bash +PROJECT_ROOT=$(cat graphify-out/.graphify_root) +# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json" +``` + +Subagent prompt template: + +See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. diff --git a/tools/skillgen/fragments/extra/.gitkeep b/tools/skillgen/fragments/extra/.gitkeep new file mode 100644 index 00000000..0276e5b8 --- /dev/null +++ b/tools/skillgen/fragments/extra/.gitkeep @@ -0,0 +1 @@ +# Reserved for later waves (codex/windows/remaining hosts). Empty in wave 2. diff --git a/tools/skillgen/fragments/extra/kilo-rules.md b/tools/skillgen/fragments/extra/kilo-rules.md new file mode 100644 index 00000000..c42881ef --- /dev/null +++ b/tools/skillgen/fragments/extra/kilo-rules.md @@ -0,0 +1,8 @@ +## Kilo-specific rules + +- Use the native `Task` tool for semantic extraction fan-out. +- Launch all chunk tasks in the same response so they run in parallel. +- Always use `subagent_type="general"` for extraction chunks. +- After modifying code files during the session, run `graphify update .`. + +--- diff --git a/tools/skillgen/fragments/extra/powershell-troubleshooting.md b/tools/skillgen/fragments/extra/powershell-troubleshooting.md new file mode 100644 index 00000000..f7a35137 --- /dev/null +++ b/tools/skillgen/fragments/extra/powershell-troubleshooting.md @@ -0,0 +1,12 @@ +## Troubleshooting + +### PowerShell 5.1: Vertical scrolling stops working + +If vertical scrolling breaks in PowerShell after running graphify, this is caused by ANSI escape sequences from the `graspologic` library. Graphify v0.3.10+ suppresses this output, but if you still see the issue: + +1. **Upgrade graphify**: `pip install --upgrade graphifyy` +2. **Use Windows Terminal** instead of the legacy PowerShell console — Windows Terminal handles ANSI codes correctly +3. **Reset your terminal**: close and reopen PowerShell +4. **Skip graspologic**: uninstall it (`pip uninstall graspologic`) and graphify will fall back to NetworkX's built-in Louvain algorithm, which produces no ANSI output + +--- diff --git a/tools/skillgen/fragments/query-stub/cli-inline.md b/tools/skillgen/fragments/query-stub/cli-inline.md new file mode 100644 index 00000000..cd080864 --- /dev/null +++ b/tools/skillgen/fragments/query-stub/cli-inline.md @@ -0,0 +1,7 @@ +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: + +```bash +graphify query "" +``` + +If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/fragments/query-stub/cli.md b/tools/skillgen/fragments/query-stub/cli.md new file mode 100644 index 00000000..3690f43c --- /dev/null +++ b/tools/skillgen/fragments/query-stub/cli.md @@ -0,0 +1,7 @@ +When `graphify-out/graph.json` already exists and the user asks a question about the corpus, run the query directly: + +```bash +graphify query "" +``` + +Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. For that vocab-expansion step, the `--dfs` / `--budget` modes, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. diff --git a/tools/skillgen/fragments/references/host/hooks-agents-md.md b/tools/skillgen/fragments/references/host/hooks-agents-md.md new file mode 100644 index 00000000..26b9a8f7 --- /dev/null +++ b/tools/skillgen/fragments/references/host/hooks-agents-md.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native AGENTS.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's AGENTS.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native AGENTS.md integration@@AGENTS_HEADING_SUFFIX@@ + +Run once per project to make graphify always-on in @@HOST_DISPLAY@@ sessions: + +```bash +@@AGENTS_INSTALL_BLOCK@@ +``` + +This writes a `## graphify` section to the local `AGENTS.md` that instructs @@HOST_DISPLAY@@ to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. +@@AGENTS_PRETOOLUSE_NOTE@@ +```bash +@@AGENTS_UNINSTALL_BLOCK@@ +``` diff --git a/tools/skillgen/fragments/references/query/cli-inline.md b/tools/skillgen/fragments/references/query/cli-inline.md new file mode 100644 index 00000000..25b826f8 --- /dev/null +++ b/tools/skillgen/fragments/references/query/cli-inline.md @@ -0,0 +1,249 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Prefer the CLI when it is installed: +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the user's verbatim question, `ANSWER` with your full answer text, and the node list with the labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/tools/skillgen/fragments/references/query/cli.md b/tools/skillgen/fragments/references/query/cli.md new file mode 100644 index 00000000..b08289e1 --- /dev/null +++ b/tools/skillgen/fragments/references/query/cli.md @@ -0,0 +1,103 @@ +# graphify reference: query, path, explain + +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +### Step 0 — Constrained query expansion (REQUIRED before traversal) + +graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. + +Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: + +1. Extract the token vocabulary from node labels: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, re +from pathlib import Path +data = json.loads(Path('graphify-out/graph.json').read_text()) +vocab = set() +for n in data['nodes']: + for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): + parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] + for p in parts: + t = p.lower() + if 3 <= len(t) <= 30: + vocab.add(t) +Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab))) +print(f'vocab: {len(vocab)} tokens') +" +``` + +2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: + - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. + - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. + - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. + - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. + - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. + +3. Print the selection explicitly to the user before running the query, so the expansion is auditable: +``` +Query expanded to (from graph vocab, N tokens): [token1, token2, ...] +``` +If the list is empty, say so plainly and stop — do not proceed to traversal. + +### Step 1 — Traversal + +Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) + +```bash +graphify query "QUESTION" +# or: graphify query "QUESTION" --dfs --budget 3000 +``` + +Answer using **only** what the graph output contains. Quote `source_location` when citing a specific fact. If the graph lacks enough information, say so - do not hallucinate edges. + +After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +```bash +graphify path "NODE_A" "NODE_B" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +```bash +graphify explain "NODE_NAME" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` diff --git a/tools/skillgen/fragments/references/shared/add-watch.md b/tools/skillgen/fragments/references/shared/add-watch.md new file mode 100644 index 00000000..78b68703 --- /dev/null +++ b/tools/skillgen/fragments/references/shared/add-watch.md @@ -0,0 +1,56 @@ +# graphify reference: add a URL and watch a folder + +Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. diff --git a/tools/skillgen/fragments/references/shared/exports.md b/tools/skillgen/fragments/references/shared/exports.md new file mode 100644 index 00000000..3750ff23 --- /dev/null +++ b/tools/skillgen/fragments/references/shared/exports.md @@ -0,0 +1,71 @@ +# graphify reference: extra exports and benchmark + +Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. + +### Step 6b - Wiki (only if --wiki flag) + +**Only run this step if `--wiki` was explicitly given in the original command.** + +Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. + +```bash +graphify export wiki +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +graphify export neo4j +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD +``` + +Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +graphify export svg +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +graphify export graphml +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +graphify benchmark +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. diff --git a/tools/skillgen/fragments/references/shared/extraction-spec-compact.md b/tools/skillgen/fragments/references/shared/extraction-spec-compact.md new file mode 100644 index 00000000..231da82b --- /dev/null +++ b/tools/skillgen/fragments/references/shared/extraction-spec-compact.md @@ -0,0 +1,29 @@ +# graphify reference: extraction subagent prompt (compact) + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation) +- INFERRED: reasonable inference (shared structure, implied dependency) +- AMBIGUOUS: uncertain — flag it, do not omit +- Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. +- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Image files: use vision — understand what the image IS, not just OCR +- DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. +- Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. +- Hyperedges: if 3+ nodes share a concept, flow, or pattern not captured by pairwise edges, add a hyperedge to a top-level `hyperedges` array. Use sparingly. Max 3 per chunk. +- If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. +- confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the immediate parent directory + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent. `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. + +Output exactly this JSON (no other text): +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +``` diff --git a/tools/skillgen/fragments/references/shared/extraction-spec.md b/tools/skillgen/fragments/references/shared/extraction-spec.md new file mode 100644 index 00000000..2bfb8733 --- /dev/null +++ b/tools/skillgen/fragments/references/shared/extraction-spec.md @@ -0,0 +1,68 @@ +# graphify reference: extraction subagent prompt + +Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH). + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. +- AMBIGUOUS edges: 0.1-0.3 + +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is `{parent_dir}_{filename_without_ext}` (the **immediate** parent directory name + the filename stem, both lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Only one level of parent is used — not the full path. Examples: `src/auth/session.py` + `ValidateToken` → `auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or the full path (e.g., `src_auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project that had ghost duplicates under the old format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. + +Generate the extraction JSON matching this schema exactly: +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): +CHUNK_PATH +``` diff --git a/tools/skillgen/fragments/references/shared/github-and-merge.md b/tools/skillgen/fragments/references/shared/github-and-merge.md new file mode 100644 index 00000000..a41ea06e --- /dev/null +++ b/tools/skillgen/fragments/references/shared/github-and-merge.md @@ -0,0 +1,46 @@ +# graphify reference: GitHub clone and cross-repo merge + +Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. + +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + +**Multiple local subfolders (monorepo or multi-service layout):** + +The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: + +```bash +graphify extract ./core/ # → ./core/graphify-out/graph.json +graphify extract ./service/ # → ./service/graphify-out/graph.json +graphify extract ./platform/ # → ./platform/graphify-out/graph.json +# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set + +# Then merge at the project root: +graphify merge-graphs \ + ./core/graphify-out/graph.json \ + ./service/graphify-out/graph.json \ + ./platform/graphify-out/graph.json \ + --out graphify-out/graph.json +``` + +Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. diff --git a/tools/skillgen/fragments/references/shared/hooks.md b/tools/skillgen/fragments/references/shared/hooks.md new file mode 100644 index 00000000..438b8b16 --- /dev/null +++ b/tools/skillgen/fragments/references/shared/hooks.md @@ -0,0 +1,33 @@ +# graphify reference: commit hook and native CLAUDE.md integration + +Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md. + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` diff --git a/tools/skillgen/fragments/references/shared/transcribe.md b/tools/skillgen/fragments/references/shared/transcribe.md new file mode 100644 index 00000000..d9cc6982 --- /dev/null +++ b/tools/skillgen/fragments/references/shared/transcribe.md @@ -0,0 +1,48 @@ +# graphify reference: transcribe video and audio + +Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** + +```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths, ensure_ascii=False)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. diff --git a/tools/skillgen/fragments/references/shared/update.md b/tools/skillgen/fragments/references/shared/update.md new file mode 100644 index 00000000..f53974a6 --- /dev/null +++ b/tools/skillgen/fragments/references/shared/update.md @@ -0,0 +1,174 @@ +# graphify reference: incremental update and cluster-only + +Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2, ensure_ascii=False)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") +deleted = list(result.get('deleted_files', [])) +if new_total == 0 and not deleted: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +if deleted: + print(f'{len(deleted)} deleted file(s) to prune.') +if new_total > 0: + print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ + 'files': r.get('new_files', {}), + 'all_files': r.get('files', {}), + 'total_files': r.get('new_total', 0), + 'total_words': r.get('total_words', 0), + 'skipped_sensitive': r.get('skipped_sensitive', []), + 'needs_graph': True, +}, ensure_ascii=False), encoding=\"utf-8\") +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + + +If no new files exist (only deletions), create an empty extraction so the merge step can prune: + +```bash +if [ ! -f graphify-out/.graphify_extract.json ]; then + echo '[graphify update] Only deletions -- creating empty extraction for merge.' + $(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') +" +fi +``` + + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.build import build_merge +from graphify.detect import save_manifest + +# Load new extraction and incremental state +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) +deleted = list(incremental.get('deleted_files', [])) + +# Use build_merge() — reads graph.json directly without NetworkX round-trip +# so edge direction (calls, implements, imports) is always preserved (#801). +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +# Write merged result back to .graphify_extract.json so Step 4 sees the full graph +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + # Explicit source/target last so they win over any stale attrs in d. + {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + # G.graph["hyperedges"] holds hyperedges from both existing graph.json + # and new_extraction (build_merge combines them). Falling back to + # new_extraction only would silently drop prior-run hyperedges (#801). + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest so next --update diffs against today's state, not the +# prior run's baseline (prevents ghost-node reports on subsequent updates). +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```bash +graphify cluster-only . +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). diff --git a/tools/skillgen/fragments/shell/posix.md b/tools/skillgen/fragments/shell/posix.md new file mode 100644 index 00000000..be3b8ab1 --- /dev/null +++ b/tools/skillgen/fragments/shell/posix.md @@ -0,0 +1,39 @@ +```bash +# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) +PYTHON="" +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +# 1. uv tool installs — most reliable on modern Mac/Linux +if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi +fi +# 2. Read shebang from graphify binary (pipx and direct pip installs) +if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$_SHEBANG" in + *[!a-zA-Z0-9/_.-]*) ;; + *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; + esac +fi +# 3. Fall back to python3 +if [ -z "$PYTHON" ]; then PYTHON="python3"; fi +if ! "$PYTHON" -c "import graphify" 2>/dev/null; then + if command -v uv >/dev/null 2>&1; then + uv tool install --upgrade graphifyy -q 2>&1 | tail -3 + _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi + else + "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ + || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 + fi +fi +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** diff --git a/tools/skillgen/fragments/shell/powershell.md b/tools/skillgen/fragments/shell/powershell.md new file mode 100644 index 00000000..dc239b50 --- /dev/null +++ b/tools/skillgen/fragments/shell/powershell.md @@ -0,0 +1,61 @@ +```powershell +# Detect Python with graphify — uv/pipx-aware (fixes #831) +New-Item -ItemType Directory -Force -Path graphify-out | Out-Null +$GRAPHIFY_PYTHON = $null + +function Find-GraphifyPython { + # 1. uv tool install — 'uv tool dir' is authoritative, respects UV_TOOL_DIR automatically + if (Get-Command uv -ErrorAction SilentlyContinue) { + $uvDir = (uv tool dir 2>$null).Trim() + if ($uvDir) { + $py = Join-Path $uvDir "graphifyy\Scripts\python.exe" + if (Test-Path $py) { + & $py -c "import graphify" 2>$null + if ($LASTEXITCODE -eq 0) { return $py } + } + } + } + # 2. pipx install — 'pipx environment' respects PIPX_HOME automatically + if (Get-Command pipx -ErrorAction SilentlyContinue) { + $venvs = (pipx environment --value PIPX_LOCAL_VENVS 2>$null).Trim() + if ($venvs) { + $py = Join-Path $venvs "graphifyy\Scripts\python.exe" + if (Test-Path $py) { + & $py -c "import graphify" 2>$null + if ($LASTEXITCODE -eq 0) { return $py } + } + } + } + # 3. Active venv / conda / pip-into-current-env + $pyCmd = Get-Command python -ErrorAction SilentlyContinue + if ($pyCmd) { + & $pyCmd.Source -c "import graphify" 2>$null + if ($LASTEXITCODE -eq 0) { + return (& $pyCmd.Source -c "import sys; print(sys.executable)").Trim() + } + } + return $null +} + +# Try to find the right Python (uv → pipx → active env) +$GRAPHIFY_PYTHON = Find-GraphifyPython + +# Not found — install then re-detect +if (-not $GRAPHIFY_PYTHON) { + if (Get-Command uv -ErrorAction SilentlyContinue) { + uv tool install --upgrade graphifyy -q 2>&1 | Select-Object -Last 3 + } else { + pip install graphifyy -q 2>&1 | Select-Object -Last 3 + } + $GRAPHIFY_PYTHON = Find-GraphifyPython +} + +# Save interpreter path — all subsequent steps read this +$GRAPHIFY_PYTHON | Out-File -FilePath graphify-out\.graphify_python -Encoding utf8 -NoNewline +# Save scan root so `graphify update` (no args) knows where to look next time +(Resolve-Path INPUT_PATH).Path | Out-File -FilePath graphify-out\.graphify_root -Encoding utf8 -NoNewline +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent block, run Python through the saved interpreter — `& (Get-Content graphify-out\.graphify_python)` in place of a bare `python3` — so every step uses the interpreter that actually has graphify.** diff --git a/tools/skillgen/gen.py b/tools/skillgen/gen.py new file mode 100644 index 00000000..b473d676 --- /dev/null +++ b/tools/skillgen/gen.py @@ -0,0 +1,897 @@ +"""skillgen: render graphify's committed skill artifacts from edited fragments. + +Build-time only. Nothing here ships in the wheel. Fragments under +``tools/skillgen/fragments/`` are the single source of truth a human edits; the +files under ``graphify/skill*.md`` and ``graphify/skills//references/`` +are generated, committed artifacts. This module renders those artifacts and +guards them against drift. + +Usage (from the repo root):: + + python -m tools.skillgen # regen every platform's artifacts + python -m tools.skillgen --platform claude + python -m tools.skillgen --check # byte-diff render vs committed + expected/, exit 1 on drift + python -m tools.skillgen --audit-coverage# per host: assert every heading of that host's own v8 body single-homes in its render + python -m tools.skillgen --schema-singleton # assert the file_type enum is byte-identical everywhere + python -m tools.skillgen --monolith-roundtrip# assert each monolith == v8 modulo the enum unification + python -m tools.skillgen --always-on-roundtrip# assert each always_on/*.md reproduces its former constant + python -m tools.skillgen --bless # rewrite expected/ from the current render + +The render is idempotent: the core template's per-platform slots are filled in a +fixed order, the reference index is sorted by name, output is LF-newline, and no +timestamp or version is ever written into a generated file. +""" +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +import tomllib +from dataclasses import dataclass, field +from pathlib import Path + +# tools/skillgen/gen.py -> repo root is two parents up. +SKILLGEN_DIR = Path(__file__).resolve().parent +REPO_ROOT = SKILLGEN_DIR.parent.parent +FRAGMENTS_DIR = SKILLGEN_DIR / "fragments" +EXPECTED_DIR = SKILLGEN_DIR / "expected" +PLATFORMS_TOML = SKILLGEN_DIR / "platforms.toml" + +# Immutable coverage baseline for --audit-coverage. The working-tree skill bodies +# are being replaced by the lean core, so the audit reads each host's v8 body +# straight from git instead of from disk. claude's v8 body is graphify/skill.md; +# every other split host has its own graphify/skill-.md. Auditing each host +# against ITS OWN v8 body is the per-host guard: a drop that only hits one host +# (e.g. trae losing its AGENTS.md integration section) is invisible when every +# host is checked against claude's monolith, so the audit must be per-host. +def _v8_baseline_ref(platform_key: str) -> str: + """The git ref for a split host's own v8 skill body.""" + if platform_key == "claude": + return "origin/v8:graphify/skill.md" + return f"origin/v8:graphify/skill-{platform_key}.md" + +# Immutable baseline for --always-on-roundtrip. The six always-on instruction +# blocks used to be triple-quoted constants in graphify/__main__.py; they are now +# packaged graphify/always_on/*.md files the module reads at load. This ref points +# at the pre-extraction source (v8, before the extraction commit on this branch) +# so the round-trip validator can prove each rendered file reproduces its former +# constant byte for byte. It deliberately does NOT track HEAD: once the extraction +# lands, HEAD's constants are _always_on(...) calls, not the literals the +# validator needs to compare against. +ALWAYS_ON_BASELINE_REF = "origin/v8:graphify/__main__.py" + +# The always-on instruction blocks: rendered-file basename -> the __main__.py +# constant it must reproduce. Rendered to graphify/always_on/.md from +# the matching fragment under fragments/always-on/. These are not platform- +# specific, so they render once in a full run (not under --platform). +ALWAYS_ON_BLOCKS = { + "claude-md": "_CLAUDE_MD_SECTION", + "agents-md": "_AGENTS_MD_SECTION", + "gemini-md": "_GEMINI_MD_SECTION", + "vscode-instructions": "_VSCODE_INSTRUCTIONS_SECTION", + "antigravity-rules": "_ANTIGRAVITY_RULES", + "kiro-steering": "_KIRO_STEERING", +} + +# The full six-value file_type enum (Decision A). Every rendered platform — split +# or monolith — must carry exactly this enum, byte for byte. schema-singleton +# guards it. +ENUM_VALUES = "code|document|paper|image|rationale|concept" +ENUM_PROSE = "`code`, `document`, `paper`, `image`, `rationale`, `concept`" + +# The eight on-demand references every split platform renders. Five are +# shared-verbatim; three (extraction-spec, query, hooks) are variant-selected and +# their source is resolved per platform from the extraction/query_variant/ +# hooks_variant fields. +_SHARED_REFERENCES = { + "update": "references/shared/update.md", + "exports": "references/shared/exports.md", + "github-and-merge": "references/shared/github-and-merge.md", + "transcribe": "references/shared/transcribe.md", + "add-watch": "references/shared/add-watch.md", +} +_EXTRACTION_SOURCE = { + "verbose": "references/shared/extraction-spec.md", + "compact": "references/shared/extraction-spec-compact.md", +} +_QUERY_SOURCE = { + "cli": "references/query/cli.md", + "cli-inline": "references/query/cli-inline.md", +} +# The hooks reference is host-flavored. Most hosts read CLAUDE.md and wire +# always-on via `graphify claude install` (the shared body). The agents-md hosts +# (trae, trae-cn, amp) read AGENTS.md and wire it via `graphify install`. +# The agents-md fragment is a per-host template: the install/uninstall commands, +# the host display name, the heading suffix, and the PreToolUse caveat are slots +# filled from _AGENTS_MD_HOOKS per host. trae carries the v8 caveat that Trae does +# NOT support PreToolUse hooks; amp's v8 had no such caveat, so its slot is empty. +# Each variant also drives the @@HOOKS_TARGET@@ pointer text in the core. The +# variant key matches the prose target file the pointer names. +_HOOKS_SOURCE = { + "claude-md": "references/shared/hooks.md", + "agents-md": "references/host/hooks-agents-md.md", +} + +# Per-host slots for the agents-md hooks reference template. Rendered EXACTLY as +# that host's v8 skill body had the "## For native AGENTS.md integration" section. +# trae's v8 heading carried a "(Trae)" suffix, the trae/trae-cn alt-command +# comments, and the no-PreToolUse-hooks Note; amp's v8 had a bare heading, a single +# install/uninstall line, and NO caveat. These are byte-faithful to v8. +_TRAE_PRETOOLUSE_NOTE = ( + "\n> **Note:** Unlike Claude Code, Trae does NOT support PreToolUse hooks. " + "The AGENTS.md rules are the always-on mechanism — there is no automatic graph " + "rebuild on tool use. Run `/graphify --update` manually after code changes if " + "the graph needs refreshing.\n" +) +_AGENTS_MD_HOOKS: dict[str, dict[str, str]] = { + "trae": { + "heading_suffix": " (Trae)", + "host_display": "Trae", + "install_block": "graphify trae install # or: graphify trae-cn install", + "uninstall_block": "graphify trae uninstall # or: graphify trae-cn uninstall # remove the section", + "pretooluse_note": _TRAE_PRETOOLUSE_NOTE, + }, + "amp": { + "heading_suffix": "", + "host_display": "Amp", + "install_block": "graphify amp install", + "uninstall_block": "graphify amp uninstall # remove the section", + "pretooluse_note": "", + }, +} +# The prose file name the lean-core hooks pointer names, per hooks variant. +_HOOKS_TARGET = { + "claude-md": "CLAUDE.md", + "agents-md": "AGENTS.md", +} + +# The v8 claude monolith (the coverage baseline) carries claude's CLI + vocab- +# expansion query design. These two sub-headings are private to that design +# (Decision C). A cli-inline platform's query reference uses the NetworkX- +# fallback traversal instead and has no vocab-expansion step, so these headings +# are legitimately absent there and must not count as a coverage hole. The +# top-level query/path/explain headings are still required everywhere. +_CLI_ONLY_QUERY_HEADINGS = { + "### Step 0 — Constrained query expansion (REQUIRED before traversal)", + "### Step 1 — Traversal", +} + +# Allowlist for the per-host coverage audit (waves 2-3 consolidations). +# +# The lean core is one shared template across every split host, so a few v8 +# headings deliberately do NOT survive verbatim in a given host's render. These +# are intentional consolidations, not content drops, and the audit must not flag +# them. Two classes: +# +# 1. SHARED_INTRO_ALLOWLIST — the lean intro consolidation. "## What graphify is +# for" is the lean intro the core carries; the minimal v8 bodies (kilo, vscode) +# had verbose intro prose with no such heading, while the richer v8 bodies +# already had it. Listing it documents the wave-2/3 intro consolidation; it +# single-homes in every render, so it is never itself a coverage hole. The enum +# unification (Decision A) is prose, not a heading, and is guarded separately by +# schema-singleton. +# +# 2. _CONSOLIDATION_ALLOWLIST[host] — per-host v8 headings the shared lean core +# re-homes under a reworded or re-leveled heading while preserving (or +# enriching) the content. The two minimal v8 bodies, kilo (414 L) and vscode +# (258 L), are the only hosts affected: the shared core is a richer superset +# that renamed their terse step/part headings and promoted kilo's +# "### Kilo-specific rules" to "## Kilo-specific rules". The mapped content +# lives in the core or a reference under the new heading; the audit confirms +# every NON-allowlisted v8 heading is single-homed, so a genuine drop (e.g. +# trae's native AGENTS.md integration) still fails loudly. +# +# Adding a heading here is a deliberate, reviewed act: it asserts "this v8 +# heading was consolidated on purpose and its content is covered elsewhere." +SHARED_INTRO_ALLOWLIST: frozenset[str] = frozenset({ + "## What graphify is for", # lean intro; v8 hosts had verbose intro prose, no heading. +}) + +_CONSOLIDATION_ALLOWLIST: dict[str, frozenset[str]] = { + # kilo's terse v8 step/part/section headings, renamed/re-leveled by the + # shared lean core. Content is preserved under the core's richer headings + # (Step 4 build/cluster/analyze, Step 5 label, Step 6 HTML, Step 9 report) + # and the query stub + references/query.md; "### Kilo-specific rules" is the + # same content promoted to "## Kilo-specific rules". + "kilo": frozenset({ + "### Step 2.5 - Transcribe video or audio files (only if video files were detected)", + "#### Part B - Semantic extraction for docs, papers, and images", + "#### Part C - Merge AST and semantic extraction", + "### Step 4 - Build the graph and generate outputs", + "### Step 5 - Save manifest, clean up, and report", + "### Query mode", + "### Kilo-specific rules", + }), + # vscode's minimal v8 step/part headings, renamed by the shared lean core. + # The build/cluster, report/visualization, and completion-summary content is + # all present under the core's Step 4/5/6/9 headings. + "vscode": frozenset({ + "#### Part A - Structural extraction (AST, free, no API cost)", + "#### Part B - Semantic extraction (AI, costs tokens)", + "### Step 4 - Build graph and cluster", + "### Step 5 - Generate report and visualization", + "### After completing all steps", + }), +} + + +def _audit_allowlist(platform_key: str) -> frozenset[str]: + """The full set of v8 headings the audit may skip for this host.""" + return SHARED_INTRO_ALLOWLIST | _CONSOLIDATION_ALLOWLIST.get(platform_key, frozenset()) + + +@dataclass(frozen=True) +class Platform: + """One render unit parsed from platforms.toml.""" + + key: str + bucket: str + skill_dst: str + # split-only template inputs + core: str | None = None + refs_dst: str | None = None + name: str = "graphify" + description: str | None = None + trigger: str | None = "/graphify" + dispatch: str | None = None + query_variant: str = "cli-inline" + extraction: str = "verbose" + shell: str = "posix" + claude_md: bool = False + hooks_variant: str = "claude-md" + extra_sections: tuple[str, ...] = () + # monolith-only inputs + monolith: str | None = None + roundtrip_ref: str | None = None + + def reference_sources(self) -> dict[str, str]: + """Resolve the rendered-name -> source-fragment map for this split platform.""" + refs = dict(_SHARED_REFERENCES) + refs["extraction-spec"] = _EXTRACTION_SOURCE[self.extraction] + refs["query"] = _QUERY_SOURCE[self.query_variant] + refs["hooks"] = _HOOKS_SOURCE[self.hooks_variant] + return refs + + @property + def hooks_target(self) -> str: + """The prose file name the lean-core hooks pointer names for this host.""" + return _HOOKS_TARGET[self.hooks_variant] + + +def load_platforms() -> dict[str, Platform]: + """Parse platforms.toml into Platform records, keyed by platform name.""" + data = tomllib.loads(PLATFORMS_TOML.read_text(encoding="utf-8")) + out: dict[str, Platform] = {} + for key, cfg in data.get("platform", {}).items(): + out[key] = Platform( + key=key, + bucket=cfg["bucket"], + skill_dst=cfg["skill_dst"], + core=cfg.get("core"), + refs_dst=cfg.get("refs_dst"), + name=cfg.get("name", "graphify"), + description=cfg.get("description"), + trigger=cfg.get("trigger", "/graphify"), + dispatch=cfg.get("dispatch"), + query_variant=cfg.get("query_variant", "cli-inline"), + extraction=cfg.get("extraction", "verbose"), + shell=cfg.get("shell", "posix"), + claude_md=bool(cfg.get("claude_md", False)), + hooks_variant=cfg.get("hooks_variant", "claude-md"), + extra_sections=tuple(cfg.get("extra_sections", [])), + monolith=cfg.get("monolith"), + roundtrip_ref=cfg.get("roundtrip_ref"), + ) + return out + + +def _read_fragment(rel: str) -> str: + """Read a fragment file under fragments/, normalised to LF newlines.""" + text = (FRAGMENTS_DIR / rel).read_text(encoding="utf-8") + return _normalise(text) + + +def _normalise(text: str) -> str: + """Force LF newlines and exactly one trailing newline.""" + text = text.replace("\r\n", "\n").replace("\r", "\n") + return text.rstrip("\n") + "\n" + + +@dataclass(frozen=True) +class RenderedArtifact: + """A single generated file: its repo-relative path and exact bytes.""" + + path: str # relative to REPO_ROOT + content: str + + +def _render_frontmatter(platform: Platform) -> str: + """Render the YAML frontmatter from the platform's name/description/trigger. + + The trigger line is omitted when the platform has no trigger (kiro/pi). + The description is preserved verbatim from platforms.toml — never invented. + """ + if platform.description is None: + raise ValueError(f"split platform '{platform.key}' is missing a description") + lines = ["---", f"name: {platform.name}", f'description: "{platform.description}"'] + if platform.trigger: + lines.append(f"trigger: {platform.trigger}") + lines.append("---") + return "\n".join(lines) + + +def _render_core(platform: Platform) -> str: + """Fill the shared core template's per-platform slots for this platform.""" + template = _read_fragment(f"core/{platform.core}.md") + + if platform.dispatch is None: + raise ValueError(f"split platform '{platform.key}' is missing a dispatch variant") + + install = _read_fragment(f"shell/{platform.shell}.md").rstrip("\n") + dispatch = _read_fragment(f"dispatch/{platform.dispatch}.md").rstrip("\n") + query_stub = _read_fragment(f"query-stub/{platform.query_variant}.md").rstrip("\n") + + if platform.extra_sections: + extra = "".join( + _read_fragment(f"extra/{name}.md").rstrip("\n") + "\n\n" + for name in platform.extra_sections + ) + else: + extra = "" + + body = ( + template.replace("@@FRONTMATTER@@", _render_frontmatter(platform)) + .replace("@@INSTALL@@", install) + .replace("@@DISPATCH@@", dispatch) + .replace("@@QUERY_STUB@@", query_stub) + .replace("@@HOOKS_TARGET@@", platform.hooks_target) + .replace("@@EXTRA@@", extra) + ) + if "@@" in body: + leftover = sorted(set(re.findall(r"@@\w+@@", body))) + raise ValueError(f"unfilled core slots for '{platform.key}': {leftover}") + return _normalise(body) + + +def _render_agents_md_hooks(platform: Platform) -> str: + """Fill the agents-md hooks template's per-host slots for this platform. + + The fragment is one template shared by every AGENTS.md host (trae, trae-cn, + amp). The install/uninstall commands, the host display name, the heading + suffix, and the PreToolUse caveat are filled from _AGENTS_MD_HOOKS so each host + renders its OWN v8 wording — trae keeps the "(Trae)" heading suffix and the + no-PreToolUse Note; amp gets a bare heading, single-line commands, and no + caveat (its v8 never had one). + """ + template = _read_fragment(_HOOKS_SOURCE["agents-md"]) + slots = _AGENTS_MD_HOOKS.get(platform.key) + if slots is None: + raise ValueError( + f"platform '{platform.key}' uses the agents-md hooks variant but has no " + f"_AGENTS_MD_HOOKS entry" + ) + body = ( + template.replace("@@AGENTS_HEADING_SUFFIX@@", slots["heading_suffix"]) + .replace("@@HOST_DISPLAY@@", slots["host_display"]) + .replace("@@AGENTS_INSTALL_BLOCK@@", slots["install_block"]) + .replace("@@AGENTS_UNINSTALL_BLOCK@@", slots["uninstall_block"]) + .replace("@@AGENTS_PRETOOLUSE_NOTE@@", slots["pretooluse_note"]) + ) + if "@@" in body: + leftover = sorted(set(re.findall(r"@@\w+@@", body))) + raise ValueError(f"unfilled agents-md hooks slots for '{platform.key}': {leftover}") + return _normalise(body) + + +def render(platform: Platform) -> list[RenderedArtifact]: + """Render every committed artifact for one platform. + + A split platform yields the lean core SKILL.md plus one file per reference, + in a stable order (core first, then references sorted by name). A monolith + yields a single inline skill body. + """ + if platform.bucket == "monolith": + body = _read_fragment(f"core/{platform.monolith}.md") + return [RenderedArtifact(platform.skill_dst, body)] + + if platform.bucket != "split": + raise ValueError(f"unknown bucket '{platform.bucket}' for platform '{platform.key}'") + + if platform.refs_dst is None: + raise ValueError(f"split platform '{platform.key}' is missing refs_dst") + + artifacts: list[RenderedArtifact] = [ + RenderedArtifact(platform.skill_dst, _render_core(platform)) + ] + + references = platform.reference_sources() + # Sorted reference index keeps the output idempotent regardless of map order. + for name in sorted(references): + # The agents-md hooks reference is a per-host template; everything else is + # read verbatim. + if name == "hooks" and platform.hooks_variant == "agents-md": + body = _render_agents_md_hooks(platform) + else: + body = _read_fragment(references[name]) + rel = f"{platform.refs_dst}/{name}.md" + artifacts.append(RenderedArtifact(rel, body)) + return artifacts + + +def render_always_on() -> list[RenderedArtifact]: + """Render the six always-on instruction blocks to graphify/always_on/*.md. + + These are the blocks the installer injects into shared files (CLAUDE.md, + AGENTS.md, GEMINI.md, .github/copilot-instructions.md, Antigravity rules, + Kiro steering). They used to be triple-quoted constants in __main__.py and + are now packaged markdown the module reads at load. Rendering them through + skillgen puts them under the --check / expected/ drift guard like every other + generated artifact. They are not platform-specific, so they render once. + """ + out: list[RenderedArtifact] = [] + for basename in sorted(ALWAYS_ON_BLOCKS): + body = _read_fragment(f"always-on/{basename}.md") + out.append(RenderedArtifact(f"graphify/always_on/{basename}.md", body)) + return out + + +def render_all(platforms: dict[str, Platform], only: str | None = None) -> list[RenderedArtifact]: + """Render the selected platforms (or all), flattened into one artifact list. + + A full render (no ``only``) also includes the always-on blocks; a single + ``--platform`` render does not, since the always-on files are shared, not + per-platform. + """ + keys = [only] if only else sorted(platforms) + out: list[RenderedArtifact] = [] + for key in keys: + if key not in platforms: + raise SystemExit(f"error: unknown platform '{key}'. Known: {', '.join(sorted(platforms))}") + out.extend(render(platforms[key])) + if only is None: + out.extend(render_always_on()) + return out + + +def write_artifacts(artifacts: list[RenderedArtifact]) -> list[str]: + """Write artifacts to disk under REPO_ROOT. Returns the paths written.""" + written: list[str] = [] + for art in artifacts: + dst = REPO_ROOT / art.path + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_text(art.content, encoding="utf-8", newline="\n") + written.append(art.path) + return written + + +def _expected_path(rel: str) -> Path: + """Map a repo-relative artifact path to its expected/ snapshot path. + + The artifact path is flattened (``/`` -> ``__``) into a single filename so + the snapshot tree never contains a ``skills/`` path component, which the + repo .gitignore ignores. This keeps expected/ a flat, fully tracked dir. + """ + return EXPECTED_DIR / (rel.replace("/", "__")) + + +def bless(artifacts: list[RenderedArtifact]) -> list[str]: + """Write the current render into expected/ as the blessed snapshot.""" + written: list[str] = [] + for art in artifacts: + dst = _expected_path(art.path) + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_text(art.content, encoding="utf-8", newline="\n") + written.append(str(dst.relative_to(SKILLGEN_DIR))) + return written + + +def check(artifacts: list[RenderedArtifact]) -> list[str]: + """Byte-diff the render against both committed artifacts and expected/. + + Returns a list of human-readable drift messages. Empty list means clean. + This is the anti-drift guard wired into CI and pre-commit: any hand-edit of + a generated file, or a stale expected/ snapshot, is caught here. + """ + problems: list[str] = [] + for art in artifacts: + committed = REPO_ROOT / art.path + if not committed.exists(): + problems.append(f"missing committed artifact: {art.path} (run: python -m tools.skillgen)") + elif committed.read_text(encoding="utf-8") != art.content: + problems.append(f"committed artifact out of date: {art.path} (run: python -m tools.skillgen)") + + snapshot = _expected_path(art.path) + if not snapshot.exists(): + problems.append(f"missing expected/ snapshot: {art.path} (run: python -m tools.skillgen --bless)") + elif snapshot.read_text(encoding="utf-8") != art.content: + problems.append(f"expected/ snapshot out of date: {art.path} (run: python -m tools.skillgen --bless)") + return problems + + +def headings(markdown: str) -> list[str]: + """Return the ATX markdown headings in source order, ignoring code fences. + + A ``#``-prefixed line inside a fenced code block is a shell comment, not a + heading, so fence state is tracked to avoid counting them. + """ + out: list[str] = [] + in_fence = False + fence_marker = "" + for line in markdown.splitlines(): + stripped = line.lstrip() + if stripped.startswith("```") or stripped.startswith("~~~"): + marker = stripped[:3] + if not in_fence: + in_fence = True + fence_marker = marker + elif marker == fence_marker: + in_fence = False + fence_marker = "" + continue + if in_fence: + continue + # An ATX heading is 1-6 '#' then a space then text. + if stripped.startswith("#"): + hashes = len(stripped) - len(stripped.lstrip("#")) + if 1 <= hashes <= 6 and stripped[hashes:hashes + 1] == " ": + out.append(stripped.strip()) + return out + + +def _git_show(ref: str) -> str: + """Read a blob from git, normalised to LF.""" + result = subprocess.run( + ["git", "show", ref], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise SystemExit(f"error: could not read {ref}: {result.stderr.strip()}") + return result.stdout + + +def _v8_available() -> bool: + """Whether origin/v8 is fetchable in this checkout. + + The git-show validators (audit-coverage, monolith-roundtrip, + always-on-roundtrip) read blobs from origin/v8. CI's default shallow checkout + does not fetch that ref, so the validators set fetch-depth: 0 to fetch it. + This probe lets the CLI skip with a clear, actionable message (rather than + crash with a cryptic git error) when the ref is genuinely unreachable. + """ + result = subprocess.run( + ["git", "rev-parse", "--verify", "--quiet", "origin/v8"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + return result.returncode == 0 + + +def audit_coverage(platform: Platform) -> list[str]: + """Assert every heading of THIS host's v8 body single-homes in its render. + + The audit reads the host's OWN v8 skill body (graphify/skill.md for claude, + graphify/skill-.md otherwise) and checks that every v8 heading lands in + that host's generated core or in exactly one of its reference fragments. This + is the per-host guard: a content drop that only hits one host (the trae native + AGENTS.md integration regression that motivated this change) is invisible when + every host is checked against claude's monolith, so each host is checked + against itself. + + Three classes of v8 heading are exempt and documented as deltas, not holes: + - the lean core's query stub re-homes claude's CLI vocab-expansion + sub-headings into the query reference (_CLI_ONLY_QUERY_HEADINGS); + - waves 2-3 consolidations (the lean "## What graphify is for" intro and the + per-host re-homed step/part headings on the minimal kilo/vscode bodies), + tracked in the audit allowlist. + Anything NOT exempt and NOT single-homed fails the audit. + """ + if platform.bucket != "split": + return [] # monoliths are guarded by the round-trip validator instead. + + problems: list[str] = [] + baseline_headings = headings(_git_show(_v8_baseline_ref(platform.key))) + allowlist = _audit_allowlist(platform.key) + + artifacts = render(platform) + by_path = {a.path: a.content for a in artifacts} + core_headings = set(headings(by_path[platform.skill_dst])) + + # Map each reference's rendered heading set. + ref_headings: dict[str, set[str]] = {} + for name in platform.reference_sources(): + rel = f"{platform.refs_dst}/{name}.md" + ref_headings[name] = set(headings(by_path[rel])) + + for h in baseline_headings: + # Allowlisted consolidations + the lean intro are intentional deltas. + if h in allowlist: + continue + # Query sub-headings that are private to the CLI + vocab-expansion design + # do not appear in a cli-inline platform's query reference (Decision C). + if platform.query_variant != "cli" and h in _CLI_ONLY_QUERY_HEADINGS: + continue + homes = [] + if h in core_headings: + homes.append("core") + for name, hs in ref_headings.items(): + if h in hs: + homes.append(f"references/{name}.md") + if not homes: + problems.append(f"v8 heading not covered anywhere: {h!r}") + elif len(homes) > 1: + problems.append(f"v8 heading double-homed in {homes}: {h!r}") + return problems + + +def _enum_lines(content: str) -> list[str]: + """Return every line in a rendered artifact that carries the file_type enum.""" + return [ + line + for line in content.splitlines() + if ENUM_VALUES in line or ENUM_PROSE in line + ] + + +# Legacy enum fragments that must never survive the six-value unification. Each +# is a strict prefix of the full superset, so a line carrying one WITHOUT the +# full superset is a stale 4- or 5-value enum. +_LEGACY_ENUMS = ( + "code|document|paper|image|rationale", # 5-value + "code|document|paper|image", # 4-value +) + + +def legacy_enum_lines(content: str) -> list[str]: + """Return lines carrying a legacy (sub-superset) file_type enum. + + A line counts as legacy only when it has a 4- or 5-value enum fragment but + NOT the full six-value superset. The schema-singleton guard treats any such + line as drift. + """ + out: list[str] = [] + for line in content.splitlines(): + if ENUM_VALUES in line: + continue + if any(bad in line for bad in _LEGACY_ENUMS): + out.append(line.strip()) + return out + + +def schema_singleton(platforms: dict[str, Platform]) -> list[str]: + """Assert the file_type enum block is byte-identical across every platform. + + Every rendered artifact that mentions the enum — the verbose and compact + extraction specs, and the inline monolith bodies — must carry exactly the + six-value superset and nothing else. A stray 4- or 5-value enum line is the + failure this guard exists to catch. + """ + problems: list[str] = [] + for key in sorted(platforms): + for art in render(platforms[key]): + for stripped in legacy_enum_lines(art.content): + problems.append( + f"[{key}] {art.path}: legacy file_type enum (not the six-value superset): {stripped!r}" + ) + return problems + + +def _is_enum_line(line: str) -> bool: + """Whether a rendered line carries the unified six-value file_type enum.""" + return ENUM_VALUES in line or ENUM_PROSE in line + + +def _is_frontmatter_description_line(line: str) -> bool: + """Whether a line is a YAML frontmatter description field. + + The unified description (graphify #1106) rewrites the frontmatter + ``description`` on every host, monoliths included. That line is now an + allowed diff against v8 alongside the enum unification. + """ + return line.lstrip().startswith("description:") + + +def monolith_roundtrip(platform: Platform) -> list[str]: + """Assert a monolith renders diff-clean vs its v8 blob modulo allowed changes. + + Two classes of line are allowed to differ between the rendered monolith and + the v8 source: the file_type enum lines (unified to the six-value superset) + and the frontmatter ``description`` line (unified across all platforms for + discovery). Every other line must match byte for byte. + """ + if platform.bucket != "monolith": + return [] + if platform.roundtrip_ref is None: + return [f"[{platform.key}] monolith is missing roundtrip_ref"] + + rendered = render(platform)[0].content + original = _normalise(_git_show(platform.roundtrip_ref)) + + rendered_lines = rendered.splitlines() + original_lines = original.splitlines() + + problems: list[str] = [] + if len(rendered_lines) != len(original_lines): + problems.append( + f"[{platform.key}] line count differs: rendered {len(rendered_lines)} vs v8 {len(original_lines)} " + "(the only allowed changes are the enum line(s) and the description line, " + "which must not add or remove lines)" + ) + return problems + + for i, (r, o) in enumerate(zip(rendered_lines, original_lines), start=1): + if r == o: + continue + # The permitted diffs are the enum unification and the unified description. + if _is_enum_line(r) or _is_frontmatter_description_line(r): + continue + problems.append( + f"[{platform.key}] line {i} differs and is not an enum or description unification:\n" + f" v8: {o!r}\n" + f" rendered: {r!r}" + ) + return problems + + +def _always_on_constants(ref: str) -> dict[str, str]: + """Parse the always-on string constants out of a __main__.py blob. + + Reads the module source from git and walks its top-level assignments, + returning ``name -> value`` for each constant in ALWAYS_ON_BLOCKS. Parsing + the source (rather than importing the live module) keeps the baseline + immutable: the validator proves fidelity against the pre-extraction text even + after the live module is rewritten to read the packaged files. + """ + import ast + + src = _git_show(ref) + wanted = set(ALWAYS_ON_BLOCKS.values()) + out: dict[str, str] = {} + for node in ast.parse(src).body: + if not isinstance(node, ast.Assign): + continue + if len(node.targets) != 1 or not isinstance(node.targets[0], ast.Name): + continue + name = node.targets[0].id + if name in wanted and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): + out[name] = node.value.value + return out + + +def always_on_roundtrip() -> list[str]: + """Assert each always_on/*.md reproduces its former constant byte for byte. + + The six always-on instruction blocks were extracted from triple-quoted + constants in __main__.py into packaged markdown. This validator renders each + block and compares it, byte for byte, against the constant's value in the + pre-extraction source (ALWAYS_ON_BASELINE_REF). A mismatch means the + extraction is not faithful and the install-string / issue-#580 contract would + break. + """ + baseline = _always_on_constants(ALWAYS_ON_BASELINE_REF) + problems: list[str] = [] + rendered = {a.path: a.content for a in render_always_on()} + for basename, const_name in sorted(ALWAYS_ON_BLOCKS.items()): + path = f"graphify/always_on/{basename}.md" + if const_name not in baseline: + problems.append(f"could not find constant {const_name} in {ALWAYS_ON_BASELINE_REF}") + continue + if rendered[path] != baseline[const_name]: + problems.append( + f"always_on/{basename}.md does not reproduce {const_name} byte for byte " + f"(rendered {len(rendered[path])} chars vs baseline {len(baseline[const_name])} chars)" + ) + return problems + + +def _parse_args(argv: list[str]) -> argparse.Namespace: + p = argparse.ArgumentParser( + prog="python -m tools.skillgen", + description="Render and guard graphify's committed skill artifacts.", + ) + p.add_argument("--platform", help="render or check just this platform key") + p.add_argument("--check", action="store_true", help="byte-diff render vs committed + expected/, exit 1 on drift") + p.add_argument("--audit-coverage", action="store_true", help="per host: assert every heading of that host's own v8 body single-homes in its render") + p.add_argument("--schema-singleton", action="store_true", help="assert the file_type enum is byte-identical everywhere") + p.add_argument("--monolith-roundtrip", action="store_true", help="assert each monolith == v8 modulo the enum unification") + p.add_argument("--always-on-roundtrip", action="store_true", help="assert each always_on/*.md reproduces its former __main__.py constant byte for byte") + p.add_argument("--bless", action="store_true", help="rewrite expected/ from the current render") + return p.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv if argv is not None else sys.argv[1:]) + platforms = load_platforms() + + # The git-show validators read origin/v8. On a shallow checkout that ref is + # absent; skip with a clear, actionable message instead of crashing. CI fixes + # this for real by setting fetch-depth: 0 so the validators actually run. + _GIT_SHOW_VALIDATORS = (args.audit_coverage, args.monolith_roundtrip, args.always_on_roundtrip) + if any(_GIT_SHOW_VALIDATORS) and not _v8_available(): + print( + "SKIPPED: origin/v8 is not fetchable in this checkout, so the git-show " + "validators cannot run. On CI, set fetch-depth: 0 on this job (actions/" + "checkout) so origin/v8 is fetched and the validators run for real.", + file=sys.stderr, + ) + return 0 + + if args.audit_coverage: + keys = [args.platform] if args.platform else sorted(platforms) + all_problems: list[str] = [] + for key in keys: + if key not in platforms: + raise SystemExit(f"error: unknown platform '{key}'") + all_problems.extend(f"[{key}] {m}" for m in audit_coverage(platforms[key])) + if all_problems: + print("audit-coverage FAILED:", file=sys.stderr) + for m in all_problems: + print(f" {m}", file=sys.stderr) + return 1 + print("audit-coverage OK: every per-host v8 heading single-homes in that host's render.") + return 0 + + if args.schema_singleton: + problems = schema_singleton( + {args.platform: platforms[args.platform]} if args.platform else platforms + ) + if problems: + print("schema-singleton FAILED (file_type enum drift):", file=sys.stderr) + for m in problems: + print(f" {m}", file=sys.stderr) + return 1 + print("schema-singleton OK: the file_type enum is the six-value superset everywhere.") + return 0 + + if args.monolith_roundtrip: + keys = [args.platform] if args.platform else sorted(platforms) + all_problems = [] + for key in keys: + all_problems.extend(monolith_roundtrip(platforms[key])) + if all_problems: + print("monolith-roundtrip FAILED:", file=sys.stderr) + for m in all_problems: + print(f" {m}", file=sys.stderr) + return 1 + print("monolith-roundtrip OK: each monolith matches v8 modulo the enum unification.") + return 0 + + if args.always_on_roundtrip: + problems = always_on_roundtrip() + if problems: + print("always-on-roundtrip FAILED:", file=sys.stderr) + for m in problems: + print(f" {m}", file=sys.stderr) + return 1 + print("always-on-roundtrip OK: each always_on/*.md reproduces its former constant byte for byte.") + return 0 + + artifacts = render_all(platforms, only=args.platform) + + if args.check: + problems = check(artifacts) + if problems: + print("check FAILED (skill artifacts have drifted):", file=sys.stderr) + for m in problems: + print(f" {m}", file=sys.stderr) + return 1 + print(f"check OK: {len(artifacts)} artifact(s) match committed output and expected/.") + return 0 + + if args.bless: + written = bless(artifacts) + print(f"blessed {len(written)} artifact(s) into expected/.") + return 0 + + written = write_artifacts(artifacts) + print(f"rendered {len(written)} artifact(s):") + for path in written: + print(f" {path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/skillgen/platforms.toml b/tools/skillgen/platforms.toml new file mode 100644 index 00000000..72842de6 --- /dev/null +++ b/tools/skillgen/platforms.toml @@ -0,0 +1,202 @@ +# skillgen platform manifest +# +# Build-time only. This file (and everything under tools/skillgen/) is NOT +# shipped in the wheel; it drives the render of the committed skill artifacts +# under graphify/. Run `python -m tools.skillgen` from the repo root to +# regenerate, `--check` to fail on drift, `--bless` to refresh expected/. +# +# Each [platform.] table declares how that host's skill is assembled. Split +# platforms share one lean core template (fragments/core/core.md) with a handful +# of per-platform slots filled from the fields below; monoliths render a single +# inline body. The shared references are assembled automatically — a platform +# declares only its deltas. +# +# Keys: +# bucket split | monolith. split = lean core + references sidecar. +# core core template basename under fragments/core/ (split only). +# skill_dst rendered SKILL.md path, relative to the repo root. +# refs_dst rendered references/ dir, relative to the repo root (split only). +# name frontmatter name (default "graphify"; graphify-windows for windows). +# description frontmatter description, PRESERVED VERBATIM per platform. Required. +# trigger frontmatter trigger (default "/graphify"; omit -> no trigger line). +# dispatch Part-B dispatch fragment basename under fragments/dispatch/. +# query_variant cli | cli-inline. Selects both the core query stub and the +# query reference body. +# extraction verbose | compact. Selects the extraction-spec reference body. +# shell posix | powershell. Selects the Step 1 install fragment. +# claude_md bool. Whether the host gets an always-on block (installer-side). +# hooks_variant claude-md | agents-md. Selects the hooks reference body and the +# lean-core hooks pointer target. Default claude-md (the host reads +# CLAUDE.md, `graphify claude install`). agents-md is for hosts that +# read AGENTS.md and wire always-on via `graphify install` +# with the no-PreToolUse-hook caveat (trae, trae-cn). +# extra_sections ordered list of extra tail fragments (under fragments/extra/), +# inserted before Honesty Rules. +# monolith core fragment basename for a monolith (monolith only). +# roundtrip_ref git ref the monolith must match modulo the enum (monolith only). + +[platform.claude] +bucket = "split" +core = "core" +skill_dst = "graphify/skill.md" +refs_dst = "graphify/skills/claude/references" +description = "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +dispatch = "agent-tool-disk" +query_variant = "cli" +extraction = "verbose" +shell = "posix" +claude_md = true + +[platform.codex] +bucket = "split" +core = "core" +skill_dst = "graphify/skill-codex.md" +refs_dst = "graphify/skills/codex/references" +description = "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +dispatch = "codex-agenttask" +query_variant = "cli-inline" +extraction = "compact" +shell = "posix" + +[platform.windows] +bucket = "split" +core = "core" +skill_dst = "graphify/skill-windows.md" +refs_dst = "graphify/skills/windows/references" +name = "graphify-windows" +description = "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +dispatch = "agent-tool-disk-powershell" +query_variant = "cli-inline" +extraction = "verbose" +shell = "powershell" +claude_md = true +extra_sections = ["powershell-troubleshooting"] + +[platform.opencode] +bucket = "split" +core = "core" +skill_dst = "graphify/skill-opencode.md" +refs_dst = "graphify/skills/opencode/references" +description = "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +dispatch = "opencode-mention" +query_variant = "cli-inline" +extraction = "verbose" + +[platform.kilo] +bucket = "split" +core = "core" +skill_dst = "graphify/skill-kilo.md" +refs_dst = "graphify/skills/kilo/references" +description = "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +dispatch = "agent-tool-disk" +query_variant = "cli-inline" +extraction = "verbose" +extra_sections = ["kilo-rules"] + +[platform.copilot] +bucket = "split" +core = "core" +skill_dst = "graphify/skill-copilot.md" +refs_dst = "graphify/skills/copilot/references" +description = "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +dispatch = "agent-tool-disk" +query_variant = "cli-inline" +extraction = "verbose" + +[platform.claw] +bucket = "split" +core = "core" +skill_dst = "graphify/skill-claw.md" +refs_dst = "graphify/skills/claw/references" +description = "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +dispatch = "agent-tool-disk" +query_variant = "cli-inline" +extraction = "compact" + +[platform.droid] +bucket = "split" +core = "core" +skill_dst = "graphify/skill-droid.md" +refs_dst = "graphify/skills/droid/references" +description = "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +dispatch = "task-tool-disk" +query_variant = "cli-inline" +extraction = "verbose" + +[platform.amp] +bucket = "split" +core = "core" +skill_dst = "graphify/skill-amp.md" +refs_dst = "graphify/skills/amp/references" +description = "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +# Amp uses the Task tool with disk-collected results (the plain task-tool-disk +# dispatch — NOT the trae variant; amp's v8 never carried the no-PreToolUse +# caveat). It reads AGENTS.md and wires always-on via `graphify amp install`, so +# it uses the agents-md hooks variant with amp's own (caveat-free) wording. +dispatch = "task-tool-disk" +query_variant = "cli-inline" +extraction = "verbose" +hooks_variant = "agents-md" + +[platform.trae] +bucket = "split" +core = "core" +skill_dst = "graphify/skill-trae.md" +refs_dst = "graphify/skills/trae/references" +description = "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +# Trae reads AGENTS.md, not CLAUDE.md, and has no PreToolUse hook. The dispatch +# block carries the no-PreToolUse caveat and the hooks reference wires +# `graphify trae install` -> AGENTS.md. trae-cn reuses this same bundle. +dispatch = "task-tool-disk-trae" +query_variant = "cli-inline" +extraction = "verbose" +hooks_variant = "agents-md" + +[platform.kiro] +bucket = "split" +core = "core" +skill_dst = "graphify/skill-kiro.md" +refs_dst = "graphify/skills/kiro/references" +description = "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +trigger = "" +dispatch = "agent-tool-disk" +query_variant = "cli-inline" +extraction = "compact" + +[platform.pi] +bucket = "split" +core = "core" +skill_dst = "graphify/skill-pi.md" +refs_dst = "graphify/skills/pi/references" +description = "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +trigger = "" +dispatch = "agent-tool-disk" +query_variant = "cli-inline" +extraction = "compact" + +[platform.vscode] +bucket = "split" +core = "core" +skill_dst = "graphify/skill-vscode.md" +refs_dst = "graphify/skills/vscode/references" +description = "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." +dispatch = "manual-paste" +query_variant = "cli-inline" +extraction = "verbose" + +# Monoliths: rendered inline as a single body, no split, no references/. The +# render must be diff-clean against the v8 blob in roundtrip_ref except for the +# file_type enum unification (5-value -> 6-value superset). monolith-roundtrip +# guards that. + +[platform.aider] +bucket = "monolith" +skill_dst = "graphify/skill-aider.md" +monolith = "aider" +roundtrip_ref = "origin/v8:graphify/skill-aider.md" + +[platform.devin] +bucket = "monolith" +skill_dst = "graphify/skill-devin.md" +monolith = "devin" +roundtrip_ref = "origin/v8:graphify/skill-devin.md"