Add --wiki export: agent-crawlable knowledge wiki from graph

This commit is contained in:
Safi
2026-04-06 16:06:31 +01:00
parent 92ab83ea38
commit d213c03adf
7 changed files with 474 additions and 17 deletions
+14
View File
@@ -1,5 +1,19 @@
# Changelog
## 0.1.8 (2026-04-05)
- Fix: follow-up questions now check for wiki first (graphify-out/wiki/index.md) before falling back to graph.json
- Fix: --update now auto-regenerates wiki if graphify-out/wiki/ exists
- Fix: community articles show truncation notice ("... and N more nodes") when > 25 nodes
- UX: pipeline completion message now lists all available flags and commands so users know what graphify can do
## 0.1.7 (2026-04-05)
- Add: `--wiki` flag — generates Wikipedia-style agent-crawlable wiki from the graph (index.md + community articles + god node articles)
- Add: `graphify/wiki.py` module with `to_wiki()` — cross-community wikilinks, cohesion scores, audit trail, navigation footer
- Add: 14 wiki tests (245 total)
- Fix: follow-up question example code now correctly splits node labels by `_` to extract verb prefixes (previous version used `def`/`fn` prefix matching which always returned zero results)
## 0.1.6 (2026-04-05)
- Fix: follow-up questions after pipeline now answered from graph.json, not by re-exploring the directory (was 25 tool calls / 1m30s; now instant)
+5
View File
@@ -14,6 +14,7 @@
graphify-out/
├── graph.html interactive graph - click nodes, search, filter by community
├── obsidian/ open as Obsidian vault
├── wiki/ Wikipedia-style articles for agent navigation (--wiki)
├── GRAPH_REPORT.md god nodes, surprising connections, suggested questions
├── graph.json persistent graph - query weeks later without re-reading
└── cache/ SHA256 cache - re-runs only process changed files
@@ -68,6 +69,8 @@ When the user types `/graphify`, invoke the Skill tool with `skill: "graphify"`
/graphify path "DigestAuth" "Response"
/graphify explain "SwinTransformer"
/graphify ./raw --watch # auto-update graph whenever files change
/graphify ./raw --wiki # build agent-crawlable wiki (index.md + article per community)
/graphify ./raw --svg # export graph.svg
/graphify ./raw --graphml # export graph.graphml (Gephi, yEd)
/graphify ./raw --neo4j # generate cypher.txt for Neo4j
@@ -93,6 +96,8 @@ Works with any mix of file types:
**Token benchmark** - printed automatically after every run. On a mixed corpus (Karpathy repos + papers + images): **71.5x** fewer tokens per query vs reading raw files.
**Wiki** (`--wiki`) - Wikipedia-style markdown articles per community and god node, with an `index.md` entry point. Point any agent at `index.md` and it can navigate the knowledge base by reading files instead of parsing JSON.
Every edge is tagged `EXTRACTED`, `INFERRED`, or `AMBIGUOUS` - you always know what was found vs guessed.
## Worked examples
+1
View File
@@ -18,6 +18,7 @@ def __getattr__(name):
"to_html": ("graphify.export", "to_html"),
"to_svg": ("graphify.export", "to_svg"),
"to_canvas": ("graphify.export", "to_canvas"),
"to_wiki": ("graphify.wiki", "to_wiki"),
}
if name in _map:
import importlib
+214
View File
@@ -0,0 +1,214 @@
# Wiki export - Wikipedia-style markdown articles from the knowledge graph
# Generates an agent-crawlable wiki: index.md + one article per community + god node articles
from __future__ import annotations
from collections import Counter
from pathlib import Path
import networkx as nx
def _safe_filename(name: str) -> str:
return name.replace("/", "-").replace(" ", "_").replace(":", "-")
def _cross_community_links(G: nx.Graph, nodes: list[str], own_cid: int, labels: dict[int, str]) -> list[tuple[str, int]]:
"""Return (community_label, edge_count) pairs for cross-community connections, sorted descending."""
counts: dict[str, int] = Counter()
for nid in nodes:
for neighbor in G.neighbors(nid):
nd = G.nodes[neighbor]
ncid = nd.get("community")
if ncid is not None and ncid != own_cid:
counts[labels.get(ncid, f"Community {ncid}")] += 1
return sorted(counts.items(), key=lambda x: -x[1])
def _community_article(
G: nx.Graph,
cid: int,
nodes: list[str],
label: str,
labels: dict[int, str],
cohesion: float | None,
) -> str:
top_nodes = sorted(nodes, key=lambda n: G.degree(n), reverse=True)[:25]
cross = _cross_community_links(G, nodes, cid, labels)
# Edge confidence breakdown
conf_counts: Counter = Counter()
for nid in nodes:
for neighbor in G.neighbors(nid):
ed = G.edges[nid, neighbor]
conf_counts[ed.get("confidence", "EXTRACTED")] += 1
total_edges = sum(conf_counts.values()) or 1
sources = sorted({G.nodes[n].get("source_file", "") for n in nodes} - {""})
lines: list[str] = []
lines += [f"# {label}", ""]
meta_parts = [f"{len(nodes)} nodes"]
if cohesion is not None:
meta_parts.append(f"cohesion {cohesion:.2f}")
lines += [f"> {' · '.join(meta_parts)}", ""]
lines += ["## Key Concepts", ""]
for nid in top_nodes:
d = G.nodes[nid]
node_label = d.get("label", nid)
src = d.get("source_file", "")
degree = G.degree(nid)
src_str = f" — `{src}`" if src else ""
lines.append(f"- **{node_label}** ({degree} connections){src_str}")
remaining = len(nodes) - len(top_nodes)
if remaining > 0:
lines.append(f"- *... and {remaining} more nodes in this community*")
lines.append("")
lines += ["## Relationships", ""]
if cross:
for other_label, count in cross[:12]:
lines.append(f"- [[{other_label}]] ({count} shared connections)")
else:
lines.append("- No strong cross-community connections detected")
lines.append("")
if sources:
lines += ["## Source Files", ""]
for src in sources[:20]:
lines.append(f"- `{src}`")
lines.append("")
lines += ["## Audit Trail", ""]
for conf in ("EXTRACTED", "INFERRED", "AMBIGUOUS"):
n = conf_counts.get(conf, 0)
pct = round(n / total_edges * 100)
lines.append(f"- {conf}: {n} ({pct}%)")
lines.append("")
lines += ["---", "", "*Part of the graphify knowledge wiki. See [[index]] to navigate.*"]
return "\n".join(lines)
def _god_node_article(G: nx.Graph, nid: str, labels: dict[int, str]) -> str:
d = G.nodes[nid]
node_label = d.get("label", nid)
src = d.get("source_file", "")
cid = d.get("community")
community_name = labels.get(cid, f"Community {cid}") if cid is not None else None
lines: list[str] = []
lines += [f"# {node_label}", ""]
lines += [f"> God node · {G.degree(nid)} connections · `{src}`", ""]
if community_name:
lines += [f"**Community:** [[{community_name}]]", ""]
# Group neighbors by relation type
by_relation: dict[str, list[str]] = {}
for neighbor in sorted(G.neighbors(nid), key=lambda n: G.degree(n), reverse=True):
nd = G.nodes[neighbor]
ed = G.edges[nid, neighbor]
rel = ed.get("relation", "related")
neighbor_label = nd.get("label", neighbor)
conf = ed.get("confidence", "")
conf_str = f" `{conf}`" if conf else ""
by_relation.setdefault(rel, []).append(f"[[{neighbor_label}]]{conf_str}")
lines += ["## Connections by Relation", ""]
for rel, targets in sorted(by_relation.items()):
lines.append(f"### {rel}")
for t in targets[:20]:
lines.append(f"- {t}")
lines.append("")
lines += ["---", "", "*Part of the graphify knowledge wiki. See [[index]] to navigate.*"]
return "\n".join(lines)
def _index_md(
communities: dict[int, list[str]],
labels: dict[int, str],
god_nodes_data: list[dict],
total_nodes: int,
total_edges: int,
) -> str:
lines: list[str] = [
"# Knowledge Graph Index",
"",
"> Auto-generated by graphify. Start here — read community articles for context, then drill into god nodes for detail.",
"",
f"**{total_nodes} nodes · {total_edges} edges · {len(communities)} communities**",
"",
"---",
"",
"## Communities",
"(sorted by size, largest first)",
"",
]
for cid, nodes in sorted(communities.items(), key=lambda x: -len(x[1])):
label = labels.get(cid, f"Community {cid}")
lines.append(f"- [[{label}]] — {len(nodes)} nodes")
lines.append("")
if god_nodes_data:
lines += ["## God Nodes", "(most connected concepts — the load-bearing abstractions)", ""]
for node in god_nodes_data:
lines.append(f"- [[{node['label']}]] — {node['edges']} connections")
lines.append("")
lines += [
"---",
"",
"*Generated by [graphify](https://github.com/safishamsi/graphify)*",
]
return "\n".join(lines)
def to_wiki(
G: nx.Graph,
communities: dict[int, list[str]],
output_dir: str | Path,
community_labels: dict[int, str] | None = None,
cohesion: dict[int, float] | None = None,
god_nodes_data: list[dict] | None = None,
) -> int:
"""Generate a Wikipedia-style wiki from the graph.
Writes:
- index.md — agent entry point, catalog of all articles
- <CommunityName>.md — one article per community
- <GodNodeLabel>.md — one article per god node
Returns the number of articles written (excluding index.md).
"""
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
labels = community_labels or {cid: f"Community {cid}" for cid in communities}
cohesion = cohesion or {}
god_nodes_data = god_nodes_data or []
count = 0
# Community articles
for cid, nodes in communities.items():
label = labels.get(cid, f"Community {cid}")
article = _community_article(G, cid, nodes, label, labels, cohesion.get(cid))
(out / f"{_safe_filename(label)}.md").write_text(article)
count += 1
# God node articles
for node_data in god_nodes_data:
nid = node_data.get("id")
if nid and nid in G:
article = _god_node_article(G, nid, labels)
(out / f"{_safe_filename(node_data['label'])}.md").write_text(article)
count += 1
# Index
(out / "index.md").write_text(
_index_md(communities, labels, god_nodes_data, G.number_of_nodes(), G.number_of_edges())
)
return count
+1 -1
View File
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "graphifyy"
version = "0.1.6"
version = "0.1.8"
description = "Claude Code skill - turn any folder of code, docs, papers, images, or tweets into a queryable knowledge graph"
readme = "README.md"
license = { text = "MIT" }
+100 -16
View File
@@ -20,6 +20,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti
/graphify <path> --html # (HTML is generated by default - this flag is a no-op)
/graphify <path> --svg # also export graph.svg (embeds in Notion, GitHub)
/graphify <path> --graphml # export graph.graphml (Gephi, yEd)
/graphify <path> --wiki # export agent-crawlable wiki (index.md + article per community + god nodes)
/graphify <path> --neo4j # generate graphify-out/cypher.txt for Neo4j
/graphify <path> --neo4j-push bolt://localhost:7687 # push directly to Neo4j
/graphify <path> --mcp # start MCP stdio server for agent access
@@ -522,7 +523,42 @@ print('graph.graphml written - open in Gephi, yEd, or any GraphML tool')
"
```
### Step 7d - MCP server (only if --mcp flag)
### Step 7d - Wiki export (only if --wiki flag)
Generates a Wikipedia-style markdown wiki: one article per community, one per god node, plus an `index.md` entry point for agents to start from. Inspired by the Farzapedia pattern — structure the knowledge so an agent can navigate it like a file system it understands.
```bash
python3 -c "
import json
from graphify.build import build_from_json
from graphify.analyze import god_nodes
from graphify.wiki import to_wiki
from pathlib import Path
extraction = json.loads(Path('.graphify_extract.json').read_text())
analysis = json.loads(Path('.graphify_analysis.json').read_text())
labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {}
G = build_from_json(extraction)
communities = {int(k): v for k, v in analysis['communities'].items()}
cohesion = {int(k): v for k, v in analysis['cohesion'].items()}
labels = {int(k): v for k, v in labels_raw.items()}
gods = god_nodes(G, top_n=20)
n = to_wiki(G, communities, 'graphify-out/wiki', community_labels=labels or None, cohesion=cohesion, god_nodes_data=gods)
print(f'Wiki: {n} articles written to graphify-out/wiki/')
print('Start at graphify-out/wiki/index.md')
"
```
The wiki contains:
- `index.md` — catalog of all communities and god nodes; agent entry point
- `<CommunityName>.md` — key concepts, cross-community links, source files, audit trail
- `<GodNodeLabel>.md` — all connections grouped by relation type, community membership
To use with an agent: point it at `index.md` and tell it to navigate the wiki to answer questions about the corpus. Works with Claude Code, Claude Desktop, or any agent that can read markdown files.
### Step 7e - MCP server (only if --mcp flag)
```bash
python3 -m graphify.serve graphify-out/graph.json
@@ -605,18 +641,25 @@ rm -f graphify-out/.needs_update 2>/dev/null || true
Tell the user:
```
Graph complete. Outputs are in a hidden folder called graphify-out/ inside the directory you ran this on.
The folder is hidden (dot prefix) so it won't show in Finder or a normal ls.
To see it:
Mac/Linux: ls -la graphify-out/
VS Code: the Explorer panel shows hidden files by default
Finder: Cmd+Shift+. to toggle hidden files
Graph complete. Outputs are in graphify-out/ inside the directory you ran this on.
What's inside:
graphify-out/obsidian/ - open this folder as a vault in Obsidian (File > Open Vault)
graphify-out/GRAPH_REPORT.md - full audit report, also readable here in Claude
graphify-out/graph.json - persistent graph, query it later with /graphify query "..."
graphify-out/obsidian/ - open as a vault in Obsidian (File > Open Vault)
graphify-out/graph.html - interactive graph, open in any browser
graphify-out/GRAPH_REPORT.md - full audit report
graphify-out/graph.json - raw graph data
What you can do next:
/graphify <path> --wiki build a Wikipedia-style wiki agents can navigate (index.md + articles)
/graphify <path> --update re-extract only new/changed files, merge into existing graph
/graphify <path> --watch auto-update graph whenever files change
/graphify add <url> fetch a URL and add it to the corpus
/graphify query "<question>" BFS search of the graph
/graphify path "ConceptA" "ConceptB" shortest path between two concepts
/graphify explain "<node>" plain-language explanation of any node
/graphify <path> --mcp start MCP server so other agents can query the graph live
/graphify <path> --neo4j export Cypher for Neo4j import
/graphify <path> --graphml export GraphML for Gephi/yEd
Full path: PATH_TO_DIR/graphify-out/
```
@@ -687,6 +730,34 @@ print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edge
Then run Steps 48 on the merged graph as normal.
After Step 8, if `graphify-out/wiki/` already exists, regenerate the wiki automatically:
```bash
python3 -c "
import json
from graphify.build import build_from_json
from graphify.analyze import god_nodes
from graphify.wiki import to_wiki
from pathlib import Path
if not Path('graphify-out/wiki').exists():
raise SystemExit(0) # wiki was never built, skip
extraction = json.loads(Path('.graphify_extract.json').read_text())
analysis = json.loads(Path('.graphify_analysis.json').read_text())
labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {}
G = build_from_json(extraction)
communities = {int(k): v for k, v in analysis['communities'].items()}
cohesion = {int(k): v for k, v in analysis['cohesion'].items()}
labels = {int(k): v for k, v in labels_raw.items()}
gods = god_nodes(G, top_n=20)
n = to_wiki(G, communities, 'graphify-out/wiki', community_labels=labels or None, cohesion=cohesion, god_nodes_data=gods)
print(f'Wiki updated: {n} articles in graphify-out/wiki/')
"
```
After Step 4, show the graph diff:
```bash
@@ -1118,7 +1189,11 @@ For the personal inspo use case: leave this running in a terminal. Drop tweets,
Do NOT use Glob, Grep, Read, Bash, or the Explore agent to answer questions about the corpus content. The graph already has the information. Re-exploring the directory defeats the entire purpose of graphify and wastes time.
Instead, load and query `graphify-out/graph.json` directly:
**If `graphify-out/wiki/index.md` exists, use the wiki — it is more readable than raw JSON.**
Start at `index.md`, read the relevant community article(s), then drill into god node articles as needed. This is faster and more accurate than parsing graph.json because the articles are already structured for agent consumption.
If the wiki does not exist, load and query `graphify-out/graph.json` directly:
```python
import json
@@ -1138,10 +1213,19 @@ Then answer using graph data:
Example — finding all verbs (action concepts) in a codebase:
```python
# Functions and methods are the verbs of code
verbs = [(d["label"], d.get("source_file", "")) for _, d in G.nodes(data=True)
if d.get("file_type") == "code" and any(k in d.get("label", "").lower()
for k in ["()", "fn ", "def ", "func"])]
from collections import Counter
# Node labels are plain names like "run", "render", "resolve" — no "def"/"fn" prefix
# Extract the first word of each function label (e.g. "load_graph" → "load")
verb_counts = Counter()
for _, d in G.nodes(data=True):
if d.get("file_type") == "code":
first_word = d.get("label", "").split("_")[0].split(".")[0].lower()
if first_word and first_word.isalpha():
verb_counts[first_word] += 1
for verb, count in verb_counts.most_common(20):
print(f"{count:>4}x {verb}")
```
**The only exception:** if the user explicitly asks you to look at a raw file (e.g., "show me the contents of X"), you may read that specific file. But for any analytical question, use the graph.
+139
View File
@@ -0,0 +1,139 @@
"""Tests for graphify.wiki — Wikipedia-style article generation."""
import pytest
from pathlib import Path
import networkx as nx
from graphify.wiki import to_wiki, _index_md, _community_article, _god_node_article
def _make_graph():
G = nx.Graph()
G.add_node("n1", label="parse", file_type="code", source_file="parser.py", community=0)
G.add_node("n2", label="validate", file_type="code", source_file="parser.py", community=0)
G.add_node("n3", label="render", file_type="code", source_file="renderer.py", community=1)
G.add_node("n4", label="stream", file_type="code", source_file="renderer.py", community=1)
G.add_edge("n1", "n2", relation="calls", confidence="EXTRACTED", weight=1.0)
G.add_edge("n1", "n3", relation="references", confidence="INFERRED", weight=1.0)
G.add_edge("n3", "n4", relation="calls", confidence="EXTRACTED", weight=1.0)
return G
COMMUNITIES = {0: ["n1", "n2"], 1: ["n3", "n4"]}
LABELS = {0: "Parsing Layer", 1: "Rendering Layer"}
COHESION = {0: 0.85, 1: 0.72}
GOD_NODES = [{"id": "n1", "label": "parse", "edges": 2}]
def test_to_wiki_writes_index(tmp_path):
G = _make_graph()
n = to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION, god_nodes_data=GOD_NODES)
assert (tmp_path / "index.md").exists()
def test_to_wiki_returns_article_count(tmp_path):
G = _make_graph()
# 2 communities + 1 god node = 3
n = to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION, god_nodes_data=GOD_NODES)
assert n == 3
def test_to_wiki_community_articles_created(tmp_path):
G = _make_graph()
to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS)
assert (tmp_path / "Parsing_Layer.md").exists()
assert (tmp_path / "Rendering_Layer.md").exists()
def test_to_wiki_god_node_article_created(tmp_path):
G = _make_graph()
to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES)
assert (tmp_path / "parse.md").exists()
def test_index_links_all_communities(tmp_path):
G = _make_graph()
to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS)
index = (tmp_path / "index.md").read_text()
assert "[[Parsing Layer]]" in index
assert "[[Rendering Layer]]" in index
def test_index_lists_god_nodes(tmp_path):
G = _make_graph()
to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES)
index = (tmp_path / "index.md").read_text()
assert "[[parse]]" in index
assert "2 connections" in index
def test_community_article_has_cross_links(tmp_path):
G = _make_graph()
to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS)
parsing = (tmp_path / "Parsing_Layer.md").read_text()
# n1 (parsing) references n3 (rendering) → cross-community link
assert "[[Rendering Layer]]" in parsing
def test_community_article_shows_cohesion(tmp_path):
G = _make_graph()
to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION)
parsing = (tmp_path / "Parsing_Layer.md").read_text()
assert "cohesion 0.85" in parsing
def test_community_article_has_audit_trail(tmp_path):
G = _make_graph()
to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS)
parsing = (tmp_path / "Parsing_Layer.md").read_text()
assert "EXTRACTED" in parsing
assert "INFERRED" in parsing
def test_god_node_article_has_connections(tmp_path):
G = _make_graph()
to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES)
article = (tmp_path / "parse.md").read_text()
assert "[[validate]]" in article or "[[render]]" in article
def test_god_node_article_links_community(tmp_path):
G = _make_graph()
to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES)
article = (tmp_path / "parse.md").read_text()
assert "[[Parsing Layer]]" in article
def test_to_wiki_skips_missing_god_node_ids(tmp_path):
"""God node with bad ID should not crash."""
G = _make_graph()
bad_gods = [{"id": "nonexistent", "label": "ghost", "edges": 99}]
n = to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=bad_gods)
# 2 communities + 0 god nodes (nonexistent skipped) = 2
assert n == 2
def test_to_wiki_no_labels_uses_fallback(tmp_path):
G = _make_graph()
to_wiki(G, COMMUNITIES, tmp_path) # no labels
assert (tmp_path / "Community_0.md").exists()
assert (tmp_path / "Community_1.md").exists()
def test_article_navigation_footer(tmp_path):
G = _make_graph()
to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS)
article = (tmp_path / "Parsing_Layer.md").read_text()
assert "[[index]]" in article
def test_community_article_truncation_notice(tmp_path):
"""Communities with more than 25 nodes show a truncation notice."""
G = nx.Graph()
nodes = [f"n{i}" for i in range(30)]
for nid in nodes:
G.add_node(nid, label=f"concept_{nid}", file_type="code", source_file="a.py", community=0)
for i in range(len(nodes) - 1):
G.add_edge(nodes[i], nodes[i + 1], relation="calls", confidence="EXTRACTED", weight=1.0)
communities = {0: nodes}
to_wiki(G, communities, tmp_path, community_labels={0: "Big Community"})
article = (tmp_path / "Big_Community.md").read_text()
assert "and 5 more nodes" in article