diff --git a/CHANGELOG.md b/CHANGELOG.md index 074e469a..4939e65d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,37 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.6.6 (2026-05-02) + +- Fix: `skill-windows.md` rewritten from PowerShell to bash — Claude Code on Windows uses git-bash so PowerShell syntax (`$null`, `$LASTEXITCODE`, `Select-Object`, `& (Get-Content ...)`, `Remove-Item`) caused exit code 49 failures; now mirrors `skill.md` structure with `python` added as fallback after `python3` for Windows Conda (#39) +- Fix: wiki `to_wiki()` now clears stale articles before regenerating, preventing orphan .md accumulation (#558) +- Fix: `_safe_filename()` in wiki.py now strips Windows-reserved characters (`< > : " / \ | ? *`) and caps length at 200 chars (#594) +- Fix: rationale-node leakage in cross-file INFERRED call resolution — rationale nodes now excluded from name lookup; edge direction (`calls`, `rationale_for`) preserved correctly at JSON export (#576) +- Feat: `.graphifyinclude` hidden path allowlist — opt specific hidden dirs into traversal (e.g. `.hermes/plans/**/*.md`) (#583) +- Feat: `--no-viz` flag wired in `cluster-only`; `GRAPHIFY_VIZ_NODE_LIMIT` env var overrides 5000-node HTML threshold (#565) +- Fix: stray colon SyntaxError in `skill-trae.md` `--cluster-only` block (#603) +- Docs: skill INFERRED confidence score guidance changed to discrete rubric (0.55/0.65/0.75/0.85/0.95) backed by calibration data (#546) +- Docs: skill `--update` prune output clarified — splits no-drift vs drift cases (#544) +- Docs: skill `--update` merge step now calls `save_manifest` to prevent deleted files reappearing (#545) + +## 0.6.5 (2026-05-02) + +- Fix: Kotlin call-walker now accepts both `simple_identifier` and `identifier` node types — PyPI's `tree_sitter_kotlin` grammar uses `identifier` while older forks use `simple_identifier`, causing zero `calls` edges to be emitted (#659) +- Feat: community sidebar now uses checkbox-based multi-select instead of show/hide buttons — supports indeterminate "select all" state (#647) +- Feat: `graphify update --force` and `GRAPHIFY_FORCE=1` env var — bypass the node-count safety check after refactors that legitimately shrink the graph (#639) +- Fix: Codex PreToolUse hook on Windows — replaced `python3 -c "..."` inline command (fails on Conda where only `python` exists, and breaks PowerShell JSON parsing) with `graphify hook-check`, a new shell-agnostic subcommand. Re-run `graphify codex install` to regenerate the hook (#651, #522) + +## 0.6.4 (2026-05-02) + +- Fix: Codex PreToolUse hook failed on Windows — `[ -f ]` is bash-only and crashes on `cmd.exe`; replaced with a cross-platform Python one-liner (`pathlib.Path.exists()`) (#651) + +## 0.6.3 (2026-05-02) + +- Fix: incremental rebuild (`graphify update`, post-commit hook) dropped INFERRED/AMBIGUOUS semantic nodes extracted from code files — node preservation now filters by ID membership in the new AST output instead of `file_type`, so LLM-extracted call/data-flow edges survive code-only rebuilds (#653) +- Fix: post-commit and post-checkout hooks blocked `git commit` for the full rebuild duration (hours on large repos) — rebuilds now detach via `nohup & disown`, git returns in ~100ms, log written to `~/.cache/graphify-rebuild.log` (#650) +- Fix: cross-file INFERRED `calls` resolution used a last-write-wins name map, causing common short names (`log`, `execute`, `find`) to accumulate hundreds of spurious edges and dominate god_nodes ranking — resolution now skips any callee name that matches 2+ candidates (ambiguous, no import evidence to pick the right target) (#543) +- Fix: `cluster-only` command crashed on graphs with >5000 nodes due to unguarded `to_html` call — now wrapped in try/except ValueError matching the watch/hook path (#541) + ## 0.6.2 (2026-05-01) - Fix: Kimi K2.6 reasoning mode consumed entire token budget leaving `content` empty — thinking now disabled on Moonshot calls so graphs actually populate (#623) diff --git a/README.md b/README.md index fbc238bc..d90124f1 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@

-**An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, Cursor, Gemini CLI, GitHub Copilot CLI, VS Code Copilot Chat, Aider, OpenClaw, Factory Droid, Trae, Hermes, Kiro, or Google Antigravity - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. +**An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, Cursor, Gemini CLI, GitHub Copilot CLI, VS Code Copilot Chat, Aider, OpenClaw, Factory Droid, Trae, Hermes, Kiro, Pi, or Google Antigravity - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. Fully multimodal. Drop in code, PDFs, markdown, screenshots, diagrams, whiteboard photos, images in other languages, or video and audio files - graphify extracts concepts and relationships from all of it and connects them into one graph. Videos are transcribed with Whisper using a domain-aware prompt derived from your corpus. YAML/YML files (Kubernetes, Kustomize, Helm, config) are indexed for semantic extraction. SQL files are AST-extracted deterministically — tables, views, functions, foreign keys, and FROM/JOIN relationships map directly into the graph with no LLM needed. 25 languages supported via tree-sitter AST (Python, JS, TS, Go, Rust, Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Lua, Zig, PowerShell, Elixir, Objective-C, Julia, Verilog, SystemVerilog, Vue, Svelte, Dart). @@ -61,7 +61,7 @@ Every relationship is tagged `EXTRACTED` (found directly in source), `INFERRED` ## Install -**Requires:** Python 3.10+ and one of: [Claude Code](https://claude.ai/code), [Codex](https://openai.com/codex), [OpenCode](https://opencode.ai), [Cursor](https://cursor.com), [Gemini CLI](https://github.com/google-gemini/gemini-cli), [GitHub Copilot CLI](https://docs.github.com/en/copilot/how-tos/copilot-cli), [VS Code Copilot Chat](https://code.visualstudio.com/docs/copilot/overview), [Aider](https://aider.chat), [OpenClaw](https://openclaw.ai), [Factory Droid](https://factory.ai), [Trae](https://trae.ai), [Kiro](https://kiro.dev), Hermes, or [Google Antigravity](https://antigravity.google) +**Requires:** Python 3.10+ and one of: [Claude Code](https://claude.ai/code), [Codex](https://openai.com/codex), [OpenCode](https://opencode.ai), [Cursor](https://cursor.com), [Gemini CLI](https://github.com/google-gemini/gemini-cli), [GitHub Copilot CLI](https://docs.github.com/en/copilot/how-tos/copilot-cli), [VS Code Copilot Chat](https://code.visualstudio.com/docs/copilot/overview), [Aider](https://aider.chat), [OpenClaw](https://openclaw.ai), [Factory Droid](https://factory.ai), [Trae](https://trae.ai), [Kiro](https://kiro.dev), [Pi](https://pi.dev), Hermes, or [Google Antigravity](https://antigravity.google) ```bash # Recommended — works on Mac and Linux with no PATH setup needed @@ -94,6 +94,7 @@ pip install graphifyy && graphify install | Gemini CLI | `graphify install --platform gemini` | | Hermes | `graphify install --platform hermes` | | Kiro IDE/CLI | `graphify kiro install` | +| Pi coding agent | `graphify install --platform pi` | | Cursor | `graphify cursor install` | | Google Antigravity | `graphify antigravity install` | @@ -127,6 +128,7 @@ After building a graph, run this once in your project: | Gemini CLI | `graphify gemini install` | | Hermes | `graphify hermes install` | | Kiro IDE/CLI | `graphify kiro install` | +| Pi coding agent | `graphify pi install` | | Google Antigravity | `graphify antigravity install` | **Claude Code** does two things: writes a `CLAUDE.md` section telling Claude to read `graphify-out/GRAPH_REPORT.md` before answering architecture questions, and installs a **PreToolUse hook** (`settings.json`) that fires before every Glob and Grep call. If a knowledge graph exists, Claude sees: _"graphify: Knowledge graph exists. Read GRAPH_REPORT.md for god nodes and community structure before searching raw files."_ — so Claude navigates via the graph instead of grepping through every file. @@ -141,6 +143,8 @@ After building a graph, run this once in your project: **Aider, OpenClaw, Factory Droid, Trae, and Hermes** write the same rules to `AGENTS.md` in your project root and copy the skill to the platform's global skill directory. These platforms don't support tool hooks, so AGENTS.md is the always-on mechanism. +**Pi coding agent** copies the skill to `~/.pi/agent/skills/graphify/SKILL.md`. Run `graphify pi install` to set it up. + **Kiro IDE/CLI** writes the skill to `.kiro/skills/graphify/SKILL.md` (invoked via `/graphify`) and a steering file to `.kiro/steering/graphify.md` with `inclusion: always` — Kiro injects this into every conversation automatically, no hook needed. **Google Antigravity** writes `.agents/rules/graphify.md` (always-on rules) and `.agents/workflows/graphify.md` (registers `/graphify` as a slash command). No hook equivalent exists in Antigravity — rules are the always-on mechanism. @@ -408,7 +412,7 @@ Audio never leaves your machine. All transcription runs locally. **Auto-sync** (`--watch`) - run in a background terminal and the graph updates itself as your codebase changes. Code file saves trigger an instant rebuild (AST only, no LLM). Doc/image changes notify you to run `--update` for the LLM re-pass. -**Git hooks** (`graphify hook install`) - installs post-commit and post-checkout hooks. Graph rebuilds automatically after every commit and every branch switch. If a rebuild fails, the hook exits with a non-zero code so git surfaces the error instead of silently continuing. No background process needed. +**Git hooks** (`graphify hook install`) - installs post-commit and post-checkout hooks. Graph rebuilds automatically after every commit and every branch switch. Rebuilds run detached in the background so `git commit` returns instantly — log written to `~/.cache/graphify-rebuild.log` (`tail -f` for status). **Wiki** (`--wiki`) - Wikipedia-style markdown articles per community and god node, with an `index.md` entry point. Point any agent at `index.md` and it can navigate the knowledge base by reading files instead of parsing JSON. diff --git a/graphify/__main__.py b/graphify/__main__.py index 2520d6b0..fb927927 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -124,6 +124,11 @@ _PLATFORM_CONFIG: dict[str, dict] = { "skill_dst": Path(".kiro") / "skills" / "graphify" / "SKILL.md", "claude_md": False, }, + "pi": { + "skill_file": "skill-pi.md", + "skill_dst": Path(".pi") / "agent" / "skills" / "graphify" / "SKILL.md", + "claude_md": False, + }, "antigravity": { "skill_file": "skill.md", "skill_dst": Path(".agents") / "skills" / "graphify" / "SKILL.md", @@ -718,11 +723,11 @@ _CODEX_HOOK = { "hooks": [ { "type": "command", - "command": ( - "[ -f graphify-out/graph.json ] && " - r"""echo '{"hookSpecificOutput":{"hookEventName":"PreToolUse","additionalContext":"graphify: Knowledge graph exists. Read graphify-out/GRAPH_REPORT.md for god nodes and community structure before searching raw files."}}' """ - "|| true" - ), + # Use the graphify CLI itself so the hook is shell-agnostic: + # no [ -f ] bash syntax, no python3 vs python Conda issue, + # no JSON escaping inside PowerShell strings. Works on + # Windows (PowerShell/cmd.exe), macOS, and Linux. + "command": "graphify hook-check", } ], } @@ -989,7 +994,7 @@ def main() -> None: print("Usage: graphify ") print() print("Commands:") - print(" install [--platform P] copy skill to platform config dir (claude|windows|codex|opencode|aider|claw|droid|trae|trae-cn|gemini|cursor|antigravity|hermes|kiro)") + print(" install [--platform P] copy skill to platform config dir (claude|windows|codex|opencode|aider|claw|droid|trae|trae-cn|gemini|cursor|antigravity|hermes|kiro|pi)") print(" path \"A\" \"B\" shortest path between two nodes in graph.json") print(" --graph path to graph.json (default graphify-out/graph.json)") print(" explain \"X\" plain-language explanation of a node and its neighbors") @@ -1005,7 +1010,10 @@ def main() -> None: print(" --dir target directory (default: ./raw)") print(" watch watch a folder and rebuild the graph on code changes") print(" update re-extract code files and update the graph (no LLM needed)") + print(" --force overwrite graph.json even if the rebuild has fewer nodes") + print(" (also: GRAPHIFY_FORCE=1 env var; use after refactors that delete code)") print(" cluster-only rerun clustering on an existing graph.json and regenerate report") + print(" --no-viz skip graph.html generation (useful for >5000 node graphs / CI)") print(" query \"\" BFS traversal of graph.json for a question") print(" --dfs use depth-first instead of breadth-first") print(" --budget N cap output at N tokens (default 2000)") @@ -1051,6 +1059,8 @@ def main() -> None: print(" hermes uninstall remove skill from ~/.hermes/skills/graphify/") print(" kiro install write skill to .kiro/skills/graphify/ + steering file (Kiro IDE/CLI)") print(" kiro uninstall remove skill + steering file") + print(" pi install write skill to ~/.pi/agent/skills/graphify/ (Pi coding agent)") + print(" pi uninstall remove skill from ~/.pi/agent/skills/graphify/") print() return @@ -1138,6 +1148,26 @@ def main() -> None: else: print("Usage: graphify kiro [install|uninstall]", file=sys.stderr) sys.exit(1) + elif cmd == "pi": + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd == "install": + install("pi") + elif subcmd == "uninstall": + skill_dst = Path.home() / ".pi" / "agent" / "skills" / "graphify" / "SKILL.md" + if skill_dst.exists(): + skill_dst.unlink() + print(f" skill removed -> {skill_dst}") + version_file = skill_dst.parent / ".graphify_version" + if version_file.exists(): + version_file.unlink() + for d in (skill_dst.parent, skill_dst.parent.parent, skill_dst.parent.parent.parent): + try: + d.rmdir() + except OSError: + break + else: + print("Usage: graphify pi [install|uninstall]", file=sys.stderr) + sys.exit(1) elif cmd in ("aider", "codex", "opencode", "claw", "droid", "trae", "trae-cn", "hermes"): subcmd = sys.argv[2] if len(sys.argv) > 2 else "" if subcmd == "install": @@ -1383,6 +1413,7 @@ def main() -> None: elif cmd == "cluster-only": watch_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(".") + no_viz = "--no-viz" in sys.argv graph_json = watch_path / "graphify-out" / "graph.json" if not graph_json.exists(): print(f"error: no graph found at {graph_json} — run /graphify first", file=sys.stderr) @@ -1412,12 +1443,34 @@ def main() -> None: out = watch_path / "graphify-out" (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") to_json(G, communities, str(out / "graph.json")) - to_html(G, communities, str(out / "graph.html"), community_labels=labels or None) - print(f"Done — {len(communities)} communities. GRAPH_REPORT.md, graph.json and graph.html updated.") + + # Mirror watch.py pattern: gate to_html so core outputs (graph.json + + # GRAPH_REPORT.md) always land. Honor --no-viz explicitly; otherwise + # fall back to ValueError handling so an oversized graph doesn't crash + # the CLI mid-write and leave a stale graph.html on disk. + html_target = out / "graph.html" + if no_viz: + if html_target.exists(): + html_target.unlink() + print(f"Done — {len(communities)} communities. GRAPH_REPORT.md and graph.json updated (--no-viz; graph.html removed).") + else: + try: + to_html(G, communities, str(html_target), community_labels=labels or None) + print(f"Done — {len(communities)} communities. GRAPH_REPORT.md, graph.json and graph.html updated.") + except ValueError as viz_err: + if html_target.exists(): + html_target.unlink() + print(f"Skipped graph.html: {viz_err}") + print(f"Done — {len(communities)} communities. GRAPH_REPORT.md and graph.json updated.") elif cmd == "update": - if len(sys.argv) > 2: - watch_path = Path(sys.argv[2]) + force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") + argv = list(sys.argv) + if "--force" in argv[2:]: + force = True + argv = [a for a in argv if a != "--force"] + if len(argv) > 2: + watch_path = Path(argv[2]) else: # Try to recover the scan root saved by the last full build saved = Path("graphify-out/.graphify_root") @@ -1430,7 +1483,7 @@ def main() -> None: sys.exit(1) from graphify.watch import _rebuild_code print(f"Re-extracting code files in {watch_path} (no LLM needed)...") - ok = _rebuild_code(watch_path) + ok = _rebuild_code(watch_path, force=force) if ok: print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.") if not os.environ.get("MOONSHOT_API_KEY") and not os.environ.get("GRAPHIFY_NO_TIPS"): @@ -1439,6 +1492,25 @@ def main() -> None: print("Nothing to update or rebuild failed — check output above.", file=sys.stderr) sys.exit(1) + elif cmd == "hook-check": + # Shell-agnostic PreToolUse hook entry point for Codex (and any platform + # where embedding Python/bash inline in a JSON hook command is fragile). + # Prints the hookSpecificOutput JSON if graph.json exists, exits 0 silently + # if not. Works on Windows PowerShell, cmd.exe, macOS, and Linux. + graph = Path("graphify-out") / "graph.json" + if graph.exists(): + import json as _json + print(_json.dumps({ + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "additionalContext": ( + "graphify: Knowledge graph exists. " + "Read graphify-out/GRAPH_REPORT.md for god nodes and " + "community structure before searching raw files." + ), + } + })) + sys.exit(0) elif cmd == "check-update": if len(sys.argv) < 3: print("Usage: graphify check-update ", file=sys.stderr) diff --git a/graphify/detect.py b/graphify/detect.py index ba1595e6..30de00e9 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -509,6 +509,115 @@ def _is_ignored(path: Path, root: Path, patterns: list[tuple[Path, str]]) -> boo return result +def _load_graphifyinclude(root: Path) -> list[tuple[Path, str]]: + """Read .graphifyinclude allowlist patterns from root and ancestors. + + Include patterns opt matching hidden files/dirs into traversal. Sensitive + files and hard-skipped noise directories are still excluded later. + Uses the same VCS-root ceiling logic as _load_graphifyignore. + """ + root = root.resolve() + ceiling = _find_vcs_root(root) or root + + dirs: list[Path] = [] + current = root + while True: + dirs.append(current) + if current == ceiling: + break + current = current.parent + dirs.reverse() + + patterns: list[tuple[Path, str]] = [] + for d in dirs: + include_file = d / ".graphifyinclude" + if include_file.exists(): + for raw in include_file.read_text(encoding="utf-8", errors="ignore").splitlines(): + line = _parse_gitignore_line(raw) + if line: + patterns.append((d, line)) + return patterns + + +def _is_included(path: Path, root: Path, patterns: list[tuple[Path, str]]) -> bool: + """Return True if path matches any .graphifyinclude allowlist pattern.""" + if not patterns: + return False + + def _matches(rel: str, p: str) -> bool: + parts = rel.split("/") + if fnmatch.fnmatch(rel, p): + return True + if fnmatch.fnmatch(path.name, p): + return True + for i, part in enumerate(parts): + if fnmatch.fnmatch(part, p): + return True + if fnmatch.fnmatch("/".join(parts[:i + 1]), p): + return True + return False + + for anchor, pattern in patterns: + anchored = pattern.startswith("/") + p = pattern.strip("/") + if not p: + continue + if anchored: + try: + rel_anchor = str(path.relative_to(anchor)).replace(os.sep, "/") + if _matches(rel_anchor, p): + return True + except ValueError: + pass + else: + try: + rel = str(path.relative_to(root)).replace(os.sep, "/") + if _matches(rel, p): + return True + except ValueError: + pass + if anchor != root: + try: + rel_anchor = str(path.relative_to(anchor)).replace(os.sep, "/") + if _matches(rel_anchor, p): + return True + except ValueError: + pass + return False + + +def _could_contain_included_path(path: Path, root: Path, patterns: list[tuple[Path, str]]) -> bool: + """Return True if a directory may contain files matched by .graphifyinclude.""" + if not patterns: + return False + + rels: list[str] = [] + try: + rels.append(str(path.relative_to(root)).replace(os.sep, "/")) + except ValueError: + pass + for anchor, _ in patterns: + if anchor != root: + try: + rels.append(str(path.relative_to(anchor)).replace(os.sep, "/")) + except ValueError: + pass + + for rel in rels: + rel = rel.strip("/") + if not rel: + return True + for _, pattern in patterns: + p = pattern.strip("/") + if not p: + continue + if p == rel or p.startswith(rel + "/"): + return True + if fnmatch.fnmatch(rel, p): + return True + return False + + def detect(root: Path, *, follow_symlinks: bool = False) -> dict: root = root.resolve() files: dict[FileType, list[str]] = { @@ -522,6 +631,7 @@ def detect(root: Path, *, follow_symlinks: bool = False) -> dict: skipped_sensitive: list[str] = [] ignore_patterns = _load_graphifyignore(root) + include_patterns = _load_graphifyinclude(root) # Always include graphify-out/memory/ - query results filed back into the graph memory_dir = root / "graphify-out" / "memory" @@ -543,10 +653,12 @@ def detect(root: Path, *, follow_symlinks: bool = False) -> dict: dirnames.clear() continue if not in_memory_tree: - # Prune noise dirs in-place so os.walk never descends into them + # Prune noise dirs in-place so os.walk never descends into them. + # Hidden dirs are allowed through if they could contain an + # explicitly included path (.graphifyinclude allowlist). dirnames[:] = [ d for d in dirnames - if not d.startswith(".") + if (not d.startswith(".") or _could_contain_included_path(dp / d, root, include_patterns)) and not _is_noise_dir(d) and not _is_ignored(dp / d, root, ignore_patterns) ] @@ -565,8 +677,9 @@ def detect(root: Path, *, follow_symlinks: bool = False) -> dict: in_memory = memory_dir.exists() and str(p).startswith(str(memory_dir)) if not in_memory: # Hidden files are already excluded via dir pruning above, - # but catch hidden files at the root level - if p.name.startswith("."): + # but catch hidden files at the root level. A .graphifyinclude + # entry can opt a specific hidden file back in. + if p.name.startswith(".") and not _is_included(p, root, include_patterns): continue # Skip files inside our own converted/ dir (avoid re-processing sidecars) if str(p).startswith(str(converted_dir)): @@ -623,7 +736,7 @@ def detect(root: Path, *, follow_symlinks: bool = False) -> dict: def _md5_file(path: Path) -> str: """MD5 of file contents streamed in 64KB chunks — for change detection only.""" import hashlib as _hl - h = _hl.md5() + h = _hl.md5(usedforsecurity=False) try: with path.open("rb") as f: for chunk in iter(lambda: f.read(65536), b""): diff --git a/graphify/export.py b/graphify/export.py index b14812fe..14587add 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -25,6 +25,22 @@ COMMUNITY_COLORS = [ MAX_NODES_FOR_VIZ = 5_000 +def _viz_node_limit() -> int: + """Return the effective viz node limit, honoring GRAPHIFY_VIZ_NODE_LIMIT env var. + + Falls back to MAX_NODES_FOR_VIZ when the env var is unset, empty, or non-integer. + Set to 0 to disable HTML viz unconditionally (useful for CI runners). + """ + import os + raw = os.environ.get("GRAPHIFY_VIZ_NODE_LIMIT") + if raw is None or not raw.strip(): + return MAX_NODES_FOR_VIZ + try: + return int(raw) + except ValueError: + return MAX_NODES_FOR_VIZ + + def _html_styles() -> str: return """""" @@ -244,26 +265,41 @@ document.addEventListener('click', e => {{ const hiddenCommunities = new Set(); +const selectAllCb = document.getElementById('select-all-cb'); + +function updateSelectAllState() {{ + const total = LEGEND.length; + const hidden = hiddenCommunities.size; + selectAllCb.checked = hidden === 0; + selectAllCb.indeterminate = hidden > 0 && hidden < total; +}} + function toggleAllCommunities(hide) {{ document.querySelectorAll('.legend-item').forEach(item => {{ hide ? item.classList.add('dimmed') : item.classList.remove('dimmed'); }}); + document.querySelectorAll('.legend-cb').forEach(cb => {{ + cb.checked = !hide; + }}); LEGEND.forEach(c => {{ if (hide) hiddenCommunities.add(c.cid); else hiddenCommunities.delete(c.cid); }}); const updates = RAW_NODES.map(n => ({{ id: n.id, hidden: hide }})); nodesDS.update(updates); + updateSelectAllState(); }} const legendEl = document.getElementById('legend'); LEGEND.forEach(c => {{ const item = document.createElement('div'); item.className = 'legend-item'; - item.innerHTML = `
- ${{c.label}} - ${{c.count}}`; - item.onclick = () => {{ - if (hiddenCommunities.has(c.cid)) {{ + const cb = document.createElement('input'); + cb.type = 'checkbox'; + cb.className = 'legend-cb'; + cb.checked = true; + cb.addEventListener('change', (e) => {{ + e.stopPropagation(); + if (cb.checked) {{ hiddenCommunities.delete(c.cid); item.classList.remove('dimmed'); }} else {{ @@ -272,8 +308,18 @@ LEGEND.forEach(c => {{ }} const updates = RAW_NODES .filter(n => n.community === c.cid) - .map(n => ({{ id: n.id, hidden: hiddenCommunities.has(c.cid) }})); + .map(n => ({{ id: n.id, hidden: !cb.checked }})); nodesDS.update(updates); + updateSelectAllState(); + }}); + item.innerHTML = `
+ ${{c.label}} + ${{c.count}}`; + item.prepend(cb); + item.onclick = (e) => {{ + if (e.target === cb) return; + cb.checked = !cb.checked; + cb.dispatchEvent(new Event('change')); }}; legendEl.appendChild(item); }}); @@ -327,6 +373,15 @@ def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str, *, if "confidence_score" not in link: conf = link.get("confidence", "EXTRACTED") link["confidence_score"] = _CONFIDENCE_SCORE_DEFAULTS.get(conf, 1.0) + # Restore original edge direction. Undirected NetworkX storage may + # canonicalize endpoint order, flipping `calls` and other directional + # edges in graph.json. The build path stashes the true endpoints in + # _src/_tgt for exactly this purpose (#563). + true_src = link.pop("_src", None) + true_tgt = link.pop("_tgt", None) + if true_src is not None and true_tgt is not None: + link["source"] = true_src + link["target"] = true_tgt data["hyperedges"] = getattr(G, "graph", {}).get("hyperedges", []) with open(output_path, "w", encoding="utf-8") as f: # nosec json.dump(data, f, indent=2) @@ -391,10 +446,12 @@ def to_html( If member_counts is provided (aggregated community view), node sizes are based on community member counts rather than graph degree. """ - if G.number_of_nodes() > MAX_NODES_FOR_VIZ: + limit = _viz_node_limit() + if G.number_of_nodes() > limit: raise ValueError( - f"Graph has {G.number_of_nodes()} nodes - too large for HTML viz. " - f"Use --no-viz or reduce input size." + f"Graph has {G.number_of_nodes()} nodes - too large for HTML viz " + f"(limit: {limit}). Use --no-viz, raise GRAPHIFY_VIZ_NODE_LIMIT, " + f"or reduce input size." ) node_community = _node_community_map(communities) @@ -431,14 +488,19 @@ def to_html( "degree": deg, }) - # Build edges list + # Build edges list. Restore original edge direction from _src/_tgt + # (stashed by build.py for exactly this reason): undirected NetworkX + # canonicalizes endpoint order, which would otherwise flip the arrow + # for `calls` and `rationale_for` in the rendered graph (#563). vis_edges = [] for u, v, data in G.edges(data=True): confidence = data.get("confidence", "EXTRACTED") relation = data.get("relation", "") + true_src = data.get("_src", u) + true_tgt = data.get("_tgt", v) vis_edges.append({ - "from": u, - "to": v, + "from": true_src, + "to": true_tgt, "label": relation, "title": _html.escape(f"{relation} [{confidence}]"), "dashes": confidence != "EXTRACTED", @@ -488,8 +550,7 @@ def to_html(

Communities

- - +
diff --git a/graphify/extract.py b/graphify/extract.py index 58c55400..5e27bf49 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -182,6 +182,7 @@ def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, s def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None: + resolved_path: "Path | None" = None for child in node.children: if child.type == "string": raw = _read_text(child, source).strip("'\"` ") @@ -197,6 +198,7 @@ def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_p elif resolved.suffix == ".jsx": resolved = resolved.with_suffix(".tsx") tgt_nid = _make_id(str(resolved)) + resolved_path = resolved else: # Check tsconfig.json path aliases (e.g. "@/" → "src/") before treating as external (#575) aliases = _load_tsconfig_aliases(Path(str_path).parent) @@ -208,6 +210,7 @@ def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_p break if resolved_alias is not None: tgt_nid = _make_id(str(resolved_alias)) + resolved_path = resolved_alias else: # Bare/scoped import (node_modules) - use last segment; dropped as external module_name = raw.split("/")[-1] @@ -225,6 +228,32 @@ def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_p }) break + # Emit symbol-level edges for named imports from local/aliased files. + # e.g. `import { Foo, type Bar } from './bar'` → file → Foo, file → Bar (EXTRACTED) + # Uses the same _make_id(target_stem, name) key that _extract_generic emits when + # defining the symbol, so these edges wire importers directly to existing symbol nodes. + if resolved_path is not None: + target_stem = _file_stem(resolved_path) + line = node.start_point[0] + 1 + for child in node.children: + if child.type == "import_clause": + for sub in child.children: + if sub.type == "named_imports": + for spec in sub.children: + if spec.type == "import_specifier": + name_node = spec.child_by_field_name("name") + if name_node: + sym = _read_text(name_node, source) + edges.append({ + "source": file_nid, + "target": _make_id(target_stem, sym), + "relation": "imports", + "confidence": "EXTRACTED", + "source_file": str_path, + "source_location": f"L{line}", + "weight": 1.0, + }) + def _import_java(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None: def _walk_scoped(n) -> str: @@ -590,7 +619,11 @@ _KOTLIN_CONFIG = LanguageConfig( call_function_field="", call_accessor_node_types=frozenset({"navigation_expression"}), call_accessor_field="", - name_fallback_child_types=("simple_identifier",), + # Different tree-sitter-kotlin grammar versions name plain identifier + # nodes differently: PyPI's `tree_sitter_kotlin` uses `identifier`, + # older forks use `simple_identifier`. Accept both so the extractor + # works across grammar generations. + name_fallback_child_types=("simple_identifier", "identifier"), body_fallback_child_types=("function_body", "class_body"), function_boundary_types=frozenset({"function_declaration"}), import_handler=_import_kotlin, @@ -1069,15 +1102,19 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict: if sc.type == "simple_identifier": callee_name = _read_text(sc, source) elif config.ts_module == "tree_sitter_kotlin": - # Kotlin: first child may be simple_identifier or navigation_expression + # Kotlin: first child may be simple_identifier/identifier or + # navigation_expression. PyPI's `tree_sitter_kotlin` produces + # `identifier` for plain identifier nodes; older grammar + # versions (including the JVM `io.github.bonede:tree-sitter-kotlin` + # binding) produce `simple_identifier`. Accept both. first = node.children[0] if node.children else None if first: - if first.type == "simple_identifier": + if first.type in ("simple_identifier", "identifier"): callee_name = _read_text(first, source) elif first.type == "navigation_expression": is_member_call = True for child in reversed(first.children): - if child.type == "simple_identifier": + if child.type in ("simple_identifier", "identifier"): callee_name = _read_text(child, source) break elif config.ts_module == "tree_sitter_scala": @@ -2830,8 +2867,17 @@ def _resolve_cross_file_imports( stem = Path(src).stem label = node.get("label", "") nid = node.get("id", "") - # Only index real classes/functions (not file nodes, not method stubs) - if label and not label.endswith((")", ".py")) and "_" not in label[:1]: + # Index class-level entities only. Function/method labels end in "()" + # so are excluded by the `endswith(")")` filter; file nodes end in ".py"; + # private/internal labels start with "_"; rationale nodes carry + # file_type=="rationale" and must never participate in cross-file + # import resolution (#563). + if ( + label + and not label.endswith((")", ".py")) + and "_" not in label[:1] + and node.get("file_type") != "rationale" + ): stem_to_entities.setdefault(stem, {})[label] = nid # Pass 2: for each file, find `from .X import A, B, C` and resolve @@ -2842,12 +2888,15 @@ def _resolve_cross_file_imports( stem = _file_stem(path) str_path = str(path) - # Find all classes defined in this file (the importers) + # Find all classes defined in this file (the importers). + # Excludes rationale nodes whose labels happen not to end in ")" or ".py" + # but which must never be treated as importing entities (#563). local_classes = [ n["id"] for n in file_result.get("nodes", []) if n.get("source_file") == str_path and not n["label"].endswith((")", ".py")) and n["id"] != _make_id(stem) # exclude file-level node + and n.get("file_type") != "rationale" ] if not local_classes: continue @@ -3697,12 +3746,21 @@ def extract( # Cross-file call resolution for all languages # Each extractor saved unresolved calls in raw_calls. Now that we have all # nodes from all files, resolve any callee that exists in another file. - global_label_to_nid: dict[str, str] = {} + # Build name → ALL matching node IDs so we can skip ambiguous common names + # (e.g. "log", "execute", "find") that appear in multiple files — resolving + # those inflates god_nodes ranking with spurious cross-file edges. + # Build label -> node_id index for cross-file call resolution. + # Skip rationale nodes (their labels are docstring text, not callable + # identifiers, and they were polluting matches for short names — #563). + global_label_to_nids: dict[str, list[str]] = {} for n in all_nodes: + if n.get("file_type") == "rationale": + continue raw = n.get("label", "") normalised = raw.strip("()").lstrip(".") if normalised: - global_label_to_nid[normalised.lower()] = n["id"] + key = normalised.lower() + global_label_to_nids.setdefault(key, []).append(n["id"]) existing_pairs = {(e["source"], e["target"]) for e in all_edges} for result in per_file: @@ -3714,9 +3772,15 @@ def extract( # and collides with any top-level function named "log" in the corpus. if rc.get("is_member_call"): continue - tgt = global_label_to_nid.get(callee.lower()) + candidates = global_label_to_nids.get(callee.lower(), []) + # Skip ambiguous names that resolve to multiple nodes — these are + # common short names (log, execute, find) with no import evidence + # to pick the right target; emitting all edges inflates god_nodes. + if len(candidates) != 1: + continue + tgt = candidates[0] caller = rc["caller_nid"] - if tgt and tgt != caller and (caller, tgt) not in existing_pairs: + if tgt != caller and (caller, tgt) not in existing_pairs: existing_pairs.add((caller, tgt)) all_edges.append({ "source": caller, diff --git a/graphify/hooks.py b/graphify/hooks.py index e921155b..eebd92ae 100644 --- a/graphify/hooks.py +++ b/graphify/hooks.py @@ -61,7 +61,13 @@ fi """ + _PYTHON_DETECT + """ export GRAPHIFY_CHANGED="$CHANGED" -$GRAPHIFY_PYTHON -c " + +# Run rebuild detached so git commit returns immediately. +# Full repo rebuilds can take hours; blocking the post-commit hook stalls the shell. +_GRAPHIFY_LOG="${HOME}/.cache/graphify-rebuild.log" +mkdir -p "$(dirname "$_GRAPHIFY_LOG")" +echo "[graphify hook] launching background rebuild (log: $_GRAPHIFY_LOG)" +nohup $GRAPHIFY_PYTHON -c " import os, sys from pathlib import Path @@ -74,12 +80,15 @@ if not changed: print(f'[graphify hook] {len(changed)} file(s) changed - rebuilding graph...') try: + import os as _os from graphify.watch import _rebuild_code - _rebuild_code(Path('.')) + _force = _os.environ.get('GRAPHIFY_FORCE', '').lower() in ('1', 'true', 'yes') + _rebuild_code(Path('.'), force=_force) except Exception as exc: print(f'[graphify hook] Rebuild failed: {exc}') sys.exit(1) -" +" > "$_GRAPHIFY_LOG" 2>&1 < /dev/null & +disown 2>/dev/null || true # graphify-hook-end """ @@ -111,17 +120,21 @@ GIT_DIR=$(git rev-parse --git-dir 2>/dev/null) [ -f "$GIT_DIR/CHERRY_PICK_HEAD" ] && exit 0 """ + _PYTHON_DETECT + """ -echo "[graphify] Branch switched - rebuilding knowledge graph (code files)..." -$GRAPHIFY_PYTHON -c " +_GRAPHIFY_LOG="${HOME}/.cache/graphify-rebuild.log" +mkdir -p "$(dirname "$_GRAPHIFY_LOG")" +echo "[graphify] Branch switched - launching background rebuild (log: $_GRAPHIFY_LOG)" +nohup $GRAPHIFY_PYTHON -c " from graphify.watch import _rebuild_code from pathlib import Path -import sys +import os, sys try: - _rebuild_code(Path('.')) + _force = os.environ.get('GRAPHIFY_FORCE', '').lower() in ('1', 'true', 'yes') + _rebuild_code(Path('.'), force=_force) except Exception as exc: print(f'[graphify] Rebuild failed: {exc}') sys.exit(1) -" +" > "$_GRAPHIFY_LOG" 2>&1 < /dev/null & +disown 2>/dev/null || true # graphify-checkout-hook-end """ diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md new file mode 100644 index 00000000..b3db4435 --- /dev/null +++ b/graphify/skill-pi.md @@ -0,0 +1,1183 @@ +--- +name: graphify +description: Turn any folder of files (code, docs, papers, images, video) into a queryable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. Use when asked to analyze a codebase, understand architecture, map dependencies, or build a knowledge graph. +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. + +Three things it does that your AI assistant alone cannot: +1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. +2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. +3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. + +Use it for: +- A codebase you're new to (understand architecture before touching anything) +- A reading list (papers + tweets + notes → one navigable graph) +- A research corpus (citation graph + concept graph in one) +- Your personal /raw folder (drop everything in, let it grow, query it) + +## What You Must Do When Invoked + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +Follow these steps in order. Do not skip steps. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles pipx, venv, system installs) +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in + *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; + esac +else + PYTHON="python3" +fi +"$PYTHON" -c "import graphify" 2>/dev/null || "$PYTHON" -m pip install graphifyy -q 2>/dev/null || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +mkdir -p graphify-out +# Write interpreter path for all subsequent steps +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat .graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat .graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result)) +" > .graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. + +**Step 2 - Transcribe:** + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('.graphify_detect.json').read_text()) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files) + Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +> **OpenClaw platform:** Multi-agent support is still early on OpenClaw. Extraction runs sequentially — you read and extract each file yourself. This is slower than parallel platforms but fully reliable. + +Print: `"Semantic extraction: N files (sequential — OpenClaw)"` + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat .graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('.graphify_detect.json').read_text()) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) +Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Sequential extraction (OpenClaw)** + +Process each file one at a time. For each file: + +1. Read the file contents +2. Extract nodes, edges, and hyperedges applying the same rules: + - EXTRACTED: relationship explicit in source (import, call, citation) + - INFERRED: reasonable inference (shared structure, implied dependency) + - AMBIGUOUS: uncertain — flag it, do not omit + - Code files: semantic edges AST cannot find. Do not re-extract imports. + - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. When adding `calls` edges: source is caller, target is callee. + - Image files: use vision — understand what the image IS, not just OCR + - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges + - Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. + - Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. + - confidence_score REQUIRED on every edge: EXTRACTED=1.0, INFERRED=0.6-0.9 (reason individually), AMBIGUOUS=0.1-0.3 +3. Accumulate results across all files + +Schema for each file's output: +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +After processing all files, write the accumulated result to `.graphify_semantic_new.json`. + +**Step B3 - Cache and merge** + +For the accumulated result: + +If more than half the chunks failed, stop and tell the user. + +Save new results to cache: +```bash +$(cat .graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `.graphify_semantic.json`: +```bash +$(cat .graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat .graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('.graphify_ast.json').read_text()) +sem = json.loads(Path('.graphify_semantic.json').read_text()) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +```bash +mkdir -p graphify-out +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +detection = json.loads(Path('.graphify_detect.json').read_text()) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +detection = json.loads(Path('.graphify_detect.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_obsidian, to_canvas +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) +print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') + +to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) +print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') +print() +print('Open graphify-out/obsidian/ as a vault in Obsidian.') +print(' Graph view - nodes colored by community (set automatically)') +print(' graph.canvas - structured layout with communities as groups') +print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') +" +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_html +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +if G.number_of_nodes() > 5000: + print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') +else: + to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) + print('graph.html written - open in any browser, no server needed') +" +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_cypher +from pathlib import Path + +G = build_from_json(json.loads(Path('.graphify_extract.json').read_text())) +to_cypher(G, 'graphify-out/cypher.txt') +print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') +" +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster +from graphify.export import push_to_neo4j +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') +" +``` + +Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_svg +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) +print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') +" +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +$(cat .graphify_python) -c " +import json +from graphify.build import build_from_json +from graphify.export import to_graphml +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +to_graphml(G, communities, 'graphify-out/graph.graphml') +print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') +" +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `.graphify_detect.json` is greater than 5,000, run: + +```bash +$(cat .graphify_python) -c " +import json +from graphify.benchmark import run_benchmark, print_benchmark +from pathlib import Path + +detection = json.loads(Path('.graphify_detect.json').read_text()) +result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) +print_benchmark(result) +" +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat .graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('.graphify_detect.json').read_text()) +save_manifest(detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('.graphify_extract.json').read_text()) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text()) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2)) + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2)) +Path('.graphify_incremental.json').write_text(json.dumps(result)) +if new_total == 0: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat .graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + +Then: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load existing graph +existing_data = json.loads(Path('graphify-out/graph.json').read_text()) +G_existing = json_graph.node_link_graph(existing_data, edges='links') + +# Load new extraction +new_extraction = json.loads(Path('.graphify_extract.json').read_text()) +G_new = build_from_json(new_extraction) + +# Merge: new nodes/edges into existing graph +G_existing.update(G_new) +print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat .graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None +new_extract = json.loads(Path('.graphify_extract.json').read_text()) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` +Clean up after: `rm -f .graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections +from graphify.report import generate +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, + 'files': {'code': [], 'document': [], 'paper': []}} +tokens = {'input': 0, 'output': 0} + +communities = cluster(G) +cohesion = score_all(G, communities) +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, +} +Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +print(f'Re-clustered: {len(communities)} communities') +" +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). + +--- + +## For /graphify query + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat .graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Load `graphify-out/graph.json`, then: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat .graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + d = G.edges[u, v] + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat .graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +First check the graph exists: +```bash +$(cat .graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +```bash +$(cat .graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + edge = G.edges[nid, path[i+1]] + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat .graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +First check the graph exists: +```bash +$(cat .graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +```bash +$(cat .graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + edge = G.edges[nid, neighbor] + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat .graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` + +--- + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat .graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. + +--- + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 2b5b401e..5f5597e6 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -826,7 +826,7 @@ G = json_graph.node_link_graph(data, edges='links') detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output':': 0} +tokens = {'input': 0, 'output': 0} communities = cluster(G) cohesion = score_all(G, communities) diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 9984022e..cca70b93 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -279,10 +279,16 @@ If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, auth confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: - EXTRACTED edges: confidence_score = 1.0 always -- INFERRED edges: reason about each edge individually. - Direct structural evidence (shared data structure, clear dependency): 0.8-0.9. - Reasonable inference with some uncertainty: 0.6-0.7. - Weak or speculative: 0.4-0.5. Most edges should be 0.6-0.9, not 0.5. +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 Output exactly this JSON (no other text): @@ -785,9 +791,28 @@ G_existing = json_graph.node_link_graph(existing_data, edges='links') new_extraction = json.loads(Path('.graphify_extract.json').read_text()) G_new = build_from_json(new_extraction) +# Prune nodes from deleted files +incremental = json.loads(Path('.graphify_incremental.json').read_text()) +deleted = set(incremental.get('deleted_files', [])) +if deleted: + to_remove = [n for n, d in G_existing.nodes(data=True) if d.get('source_file') in deleted] + G_existing.remove_nodes_from(to_remove) + if to_remove: + print(f'Pruned {len(to_remove)} ghost node(s) from {len(deleted)} deleted file(s) — drift detected and corrected.') + else: + print(f'{len(deleted)} file(s) deleted since last run, but no ghost nodes were present in the graph — no drift.') + # Merge: new nodes/edges into existing graph G_existing.update(G_new) print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') + +# Save manifest with the CURRENT full file list so the next --update +# diffs against today's filesystem state, not the prior --update's +# baseline. Without this, deleted files get reported as ghosts again +# on every subsequent --update until a full rebuild runs. +from graphify.detect import save_manifest +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') " ``` diff --git a/graphify/skill.md b/graphify/skill.md index 10e4b701..c49cbfb3 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -337,10 +337,16 @@ If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, auth confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: - EXTRACTED edges: confidence_score = 1.0 always -- INFERRED edges: reason about each edge individually. - Direct structural evidence (shared data structure, clear dependency): 0.8-0.9. - Reasonable inference with some uncertainty: 0.6-0.7. - Weak or speculative: 0.4-0.5. Most edges should be 0.6-0.9, not 0.5. +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the filename without extension and entity is the symbol name, both normalized (lowercase, non-alphanumeric chars replaced with `_`). Example: `src/auth/session.py` + `ValidateToken` → `session_validatetoken`. This must match the ID the AST extractor generates so cross-references between code and semantic nodes connect correctly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. @@ -923,7 +929,10 @@ deleted = set(incremental.get('deleted_files', [])) if deleted: to_remove = [n for n, d in G_existing.nodes(data=True) if d.get('source_file') in deleted] G_existing.remove_nodes_from(to_remove) - print(f'Pruned {len(to_remove)} ghost nodes from {len(deleted)} deleted file(s)') + if to_remove: + print(f'Pruned {len(to_remove)} ghost node(s) from {len(deleted)} deleted file(s) — drift detected and corrected.') + else: + print(f'{len(deleted)} file(s) deleted since last run, but no ghost nodes were present in the graph — no drift.') # Merge: new nodes/edges into existing graph G_existing.update(G_new) @@ -939,6 +948,14 @@ merged_out = { } Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out)) print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest with the CURRENT full file list so the next --update +# diffs against today's filesystem state, not the prior --update's +# baseline. Without this, deleted files get reported as ghosts again +# on every subsequent --update until a full rebuild runs. +from graphify.detect import save_manifest +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') " ``` diff --git a/graphify/transcribe.py b/graphify/transcribe.py index 701fdb4c..6d21038f 100644 --- a/graphify/transcribe.py +++ b/graphify/transcribe.py @@ -58,7 +58,7 @@ def download_audio(url: str, output_dir: Path) -> Path: # yt-dlp uses %(title)s which can be long/weird — use a stable name based on URL hash import hashlib - url_hash = hashlib.sha1(url.encode()).hexdigest()[:12] + url_hash = hashlib.sha1(url.encode(), usedforsecurity=False).hexdigest()[:12] out_template = str(output_dir / f"yt_{url_hash}.%(ext)s") # Check for already-downloaded file diff --git a/graphify/watch.py b/graphify/watch.py index 1e07bd39..12496851 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -33,9 +33,13 @@ def _relativize_source_files(payload: dict, root: Path) -> None: continue -def _rebuild_code(watch_path: Path, *, follow_symlinks: bool = False) -> bool: +def _rebuild_code(watch_path: Path, *, follow_symlinks: bool = False, force: bool = False) -> bool: """Re-run AST extraction + build + cluster + report for code files. No LLM needed. + When ``force`` is True the node-count safety check in ``to_json`` is bypassed + so the rebuilt graph overwrites graph.json even if it has fewer nodes. + Use this after refactors that legitimately delete code. + Returns True on success, False on error. """ watch_root = watch_path.resolve() @@ -60,20 +64,25 @@ def _rebuild_code(watch_path: Path, *, follow_symlinks: bool = False) -> bool: result = extract(code_files, cache_root=watch_root) # Preserve semantic nodes/edges from a previous full run. - # AST-only rebuild replaces code nodes; doc/paper/image nodes are kept. + # AST-only rebuild replaces nodes for changed files; everything else is kept. + # Filter by node ID membership in the new AST output, not by file_type — + # INFERRED/AMBIGUOUS nodes extracted from code files also carry file_type="code" + # and would be wrongly dropped by a file_type-based filter. out = watch_path / "graphify-out" existing_graph = out / "graph.json" if existing_graph.exists(): try: existing = json.loads(existing_graph.read_text(encoding="utf-8")) - code_ids = {n["id"] for n in existing.get("nodes", []) if n.get("file_type") == "code"} - sem_nodes = [n for n in existing.get("nodes", []) if n.get("file_type") != "code"] - sem_edges = [e for e in existing.get("links", existing.get("edges", [])) - if e.get("confidence") in ("INFERRED", "AMBIGUOUS") - or (e.get("source") not in code_ids and e.get("target") not in code_ids)] + new_ast_ids = {n["id"] for n in result["nodes"]} + preserved_nodes = [n for n in existing.get("nodes", []) if n["id"] not in new_ast_ids] + all_ids = new_ast_ids | {n["id"] for n in preserved_nodes} + preserved_edges = [ + e for e in existing.get("links", existing.get("edges", [])) + if e.get("source") in all_ids and e.get("target") in all_ids + ] result = { - "nodes": result["nodes"] + sem_nodes, - "edges": result["edges"] + sem_edges, + "nodes": result["nodes"] + preserved_nodes, + "edges": result["edges"] + preserved_edges, "hyperedges": existing.get("hyperedges", []), "input_tokens": 0, "output_tokens": 0, @@ -100,7 +109,7 @@ def _rebuild_code(watch_path: Path, *, follow_symlinks: bool = False) -> bool: out.mkdir(exist_ok=True) (out / ".graphify_root").write_text(str(watch_root), encoding="utf-8") - json_written = to_json(G, communities, str(out / "graph.json")) + json_written = to_json(G, communities, str(out / "graph.json"), force=force) if not json_written: return False diff --git a/graphify/wiki.py b/graphify/wiki.py index 6b2f5f95..8cc845f7 100644 --- a/graphify/wiki.py +++ b/graphify/wiki.py @@ -7,7 +7,18 @@ import networkx as nx def _safe_filename(name: str) -> str: - return name.replace("/", "-").replace(" ", "_").replace(":", "-") + """Make a label safe for use as a filename across platforms. + + Substitutes characters that Windows reserves in filenames + (< > : " / \\ | ? *) and strips trailing dots/spaces, also reserved. + Falls back to 'unnamed' for empty results and caps length at 200 + chars to stay well under common filesystem limits. + """ + import re + s = name.replace("/", "-").replace(" ", "_").replace(":", "-") + s = re.sub(r'[<>:"/\\|?*]', '_', s) + s = s.strip('. ') + return s[:200] if s else 'unnamed' def _cross_community_links(G: nx.Graph, nodes: list[str], own_cid: int, labels: dict[int, str]) -> list[tuple[str, int]]: @@ -185,6 +196,15 @@ def to_wiki( out = Path(output_dir) out.mkdir(parents=True, exist_ok=True) + # Clear stale .md files from previous runs to prevent orphan accumulation. + # Community labels are LLM-generated (per skill.md Step 5) and non-deterministic + # across runs — the same conceptual community may be named differently each time + # (e.g. "AutoAgent Skills" → "AutoAgent Methodology"), leaving the previous file + # as an orphan. Since to_wiki() owns wiki/ entirely (always writes the full set), + # it can safely clear .md files at the start of each call. + for old_article in out.glob("*.md"): + old_article.unlink() + labels = community_labels or {cid: f"Community {cid}" for cid in communities} cohesion = cohesion or {} god_nodes_data = god_nodes_data or [] diff --git a/pyproject.toml b/pyproject.toml index ee2b7596..fd65161a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.6.2" -description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, Gemini CLI, Aider, OpenClaw, Factory Droid, Trae, Hermes, Kiro, Google Antigravity) - turn any folder of code, docs, papers, images, or videos into a queryable knowledge graph" +version = "0.6.6" +description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, Gemini CLI, Aider, OpenClaw, Factory Droid, Trae, Hermes, Kiro, Pi, Google Antigravity) - turn any folder of code, docs, papers, images, or videos into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } -keywords = ["claude", "claude-code", "codex", "opencode", "cursor", "gemini", "aider", "kiro", "knowledge-graph", "rag", "graphrag", "obsidian", "community-detection", "tree-sitter", "leiden", "llm"] +keywords = ["claude", "claude-code", "codex", "opencode", "cursor", "gemini", "aider", "kiro", "pi", "knowledge-graph", "rag", "graphrag", "obsidian", "community-detection", "tree-sitter", "leiden", "llm"] requires-python = ">=3.10" dependencies = [ "networkx", @@ -62,9 +62,9 @@ graphify = "graphify.__main__:main" # Run without installing: uvx graphifyy install package = true -[tool.setuptools.packages.find] -where = ["."] -include = ["graphify*"] +[tool.setuptools] +packages = ["graphify"] +include-package-data = false [tool.setuptools.package-data] -graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-aider.md", "skill-copilot.md", "skill-claw.md", "skill-windows.md", "skill-droid.md", "skill-trae.md", "skill-kiro.md", "skill-vscode.md"] +graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-aider.md", "skill-copilot.md", "skill-claw.md", "skill-windows.md", "skill-droid.md", "skill-trae.md", "skill-kiro.md", "skill-vscode.md", "skill-pi.md"] diff --git a/tests/test_extract.py b/tests/test_extract.py index 3d5b9f53..3a26693d 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -168,3 +168,25 @@ def test_calls_deduplication(): result = extract_python(FIXTURES / "sample_calls.py") call_pairs = [(e["source"], e["target"]) for e in result["edges"] if e["relation"] == "calls"] assert len(call_pairs) == len(set(call_pairs)), "Duplicate calls edges found" + + +def test_cross_file_calls_skip_ambiguous_duplicate_labels(tmp_path): + """Unqualified cross-file calls must not guess between duplicate helper names.""" + caller = tmp_path / "caller.py" + helper_a = tmp_path / "a.py" + helper_b = tmp_path / "b.py" + caller.write_text("def run():\n log()\n") + helper_a.write_text("def log():\n return 'a'\n") + helper_b.write_text("def log():\n return 'b'\n") + + result = extract([caller, helper_a, helper_b], cache_root=tmp_path) + nodes = {n["id"]: n for n in result["nodes"]} + calls = [ + e for e in result["edges"] + if e["relation"] == "calls" and e["confidence"] == "INFERRED" + ] + + assert not any( + nodes[e["source"]]["label"] == "run()" and nodes[e["target"]]["label"] == "log()" + for e in calls + ) diff --git a/tests/test_languages.py b/tests/test_languages.py index 680bb4e2..c9150f78 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -188,6 +188,18 @@ def test_kotlin_finds_function(): r = extract_kotlin(FIXTURES / "sample.kt") assert any("createClient" in l for l in _labels(r)) +def test_kotlin_emits_in_file_calls(): + """Regression test for the call-walker `simple_identifier` / + `identifier` rename — see graphify-kmp's PythonParityTest.""" + r = extract_kotlin(FIXTURES / "sample.kt") + calls = _calls(r) + # In sample.kt: get() and post() both call buildRequest(), and + # createClient() invokes Config and HttpClient (constructor calls). + assert (".get()", ".buildRequest()") in calls + assert (".post()", ".buildRequest()") in calls + assert ("createClient()", "Config") in calls + assert ("createClient()", "HttpClient") in calls + # ── Scala ─────────────────────────────────────────────────────────────────────