Add --wiki export: agent-crawlable knowledge wiki from graph

2026-07-14 11:27:10 +00:00 · 2026-04-05 16:54:50 +01:00
parent 93b8ec3cde
commit ef06d7d8d9
6 changed files with 395 additions and 6 deletions
@@ -1,5 +1,12 @@
 # Changelog

+## 0.1.7 (2026-04-05)
+
+- Add: `--wiki` flag — generates Wikipedia-style agent-crawlable wiki from the graph (index.md + community articles + god node articles)
+- Add: `graphify/wiki.py` module with `to_wiki()` — cross-community wikilinks, cohesion scores, audit trail, navigation footer
+- Add: 14 wiki tests (245 total)
+- Fix: follow-up question example code now correctly splits node labels by `_` to extract verb prefixes (previous version used `def`/`fn` prefix matching which always returned zero results)
+
 ## 0.1.6 (2026-04-05)

 - Fix: follow-up questions after pipeline now answered from graph.json, not by re-exploring the directory (was 25 tool calls / 1m30s; now instant)
@@ -18,6 +18,7 @@ def __getattr__(name):
        "to_html": ("graphify.export", "to_html"),
        "to_svg": ("graphify.export", "to_svg"),
        "to_canvas": ("graphify.export", "to_canvas"),
+        "to_wiki": ("graphify.wiki", "to_wiki"),
    }
    if name in _map:
        import importlib
@@ -0,0 +1,211 @@
+# Wiki export - Wikipedia-style markdown articles from the knowledge graph
+# Generates an agent-crawlable wiki: index.md + one article per community + god node articles
+from __future__ import annotations
+from collections import Counter
+from pathlib import Path
+import networkx as nx
+
+
+def _safe_filename(name: str) -> str:
+    return name.replace("/", "-").replace(" ", "_").replace(":", "-")
+
+
+def _cross_community_links(G: nx.Graph, nodes: list[str], own_cid: int, labels: dict[int, str]) -> list[tuple[str, int]]:
+    """Return (community_label, edge_count) pairs for cross-community connections, sorted descending."""
+    counts: dict[str, int] = Counter()
+    for nid in nodes:
+        for neighbor in G.neighbors(nid):
+            nd = G.nodes[neighbor]
+            ncid = nd.get("community")
+            if ncid is not None and ncid != own_cid:
+                counts[labels.get(ncid, f"Community {ncid}")] += 1
+    return sorted(counts.items(), key=lambda x: -x[1])
+
+
+def _community_article(
+    G: nx.Graph,
+    cid: int,
+    nodes: list[str],
+    label: str,
+    labels: dict[int, str],
+    cohesion: float | None,
+) -> str:
+    top_nodes = sorted(nodes, key=lambda n: G.degree(n), reverse=True)[:25]
+    cross = _cross_community_links(G, nodes, cid, labels)
+
+    # Edge confidence breakdown
+    conf_counts: Counter = Counter()
+    for nid in nodes:
+        for neighbor in G.neighbors(nid):
+            ed = G.edges[nid, neighbor]
+            conf_counts[ed.get("confidence", "EXTRACTED")] += 1
+    total_edges = sum(conf_counts.values()) or 1
+
+    sources = sorted({G.nodes[n].get("source_file", "") for n in nodes} - {""})
+
+    lines: list[str] = []
+    lines += [f"# {label}", ""]
+
+    meta_parts = [f"{len(nodes)} nodes"]
+    if cohesion is not None:
+        meta_parts.append(f"cohesion {cohesion:.2f}")
+    lines += [f"> {' · '.join(meta_parts)}", ""]
+
+    lines += ["## Key Concepts", ""]
+    for nid in top_nodes:
+        d = G.nodes[nid]
+        node_label = d.get("label", nid)
+        src = d.get("source_file", "")
+        degree = G.degree(nid)
+        src_str = f" — `{src}`" if src else ""
+        lines.append(f"- **{node_label}** ({degree} connections){src_str}")
+    lines.append("")
+
+    lines += ["## Relationships", ""]
+    if cross:
+        for other_label, count in cross[:12]:
+            lines.append(f"- [[{other_label}]] ({count} shared connections)")
+    else:
+        lines.append("- No strong cross-community connections detected")
+    lines.append("")
+
+    if sources:
+        lines += ["## Source Files", ""]
+        for src in sources[:20]:
+            lines.append(f"- `{src}`")
+        lines.append("")
+
+    lines += ["## Audit Trail", ""]
+    for conf in ("EXTRACTED", "INFERRED", "AMBIGUOUS"):
+        n = conf_counts.get(conf, 0)
+        pct = round(n / total_edges * 100)
+        lines.append(f"- {conf}: {n} ({pct}%)")
+    lines.append("")
+
+    lines += ["---", "", "*Part of the graphify knowledge wiki. See [[index]] to navigate.*"]
+    return "\n".join(lines)
+
+
+def _god_node_article(G: nx.Graph, nid: str, labels: dict[int, str]) -> str:
+    d = G.nodes[nid]
+    node_label = d.get("label", nid)
+    src = d.get("source_file", "")
+    cid = d.get("community")
+    community_name = labels.get(cid, f"Community {cid}") if cid is not None else None
+
+    lines: list[str] = []
+    lines += [f"# {node_label}", ""]
+    lines += [f"> God node · {G.degree(nid)} connections · `{src}`", ""]
+
+    if community_name:
+        lines += [f"**Community:** [[{community_name}]]", ""]
+
+    # Group neighbors by relation type
+    by_relation: dict[str, list[str]] = {}
+    for neighbor in sorted(G.neighbors(nid), key=lambda n: G.degree(n), reverse=True):
+        nd = G.nodes[neighbor]
+        ed = G.edges[nid, neighbor]
+        rel = ed.get("relation", "related")
+        neighbor_label = nd.get("label", neighbor)
+        conf = ed.get("confidence", "")
+        conf_str = f" `{conf}`" if conf else ""
+        by_relation.setdefault(rel, []).append(f"[[{neighbor_label}]]{conf_str}")
+
+    lines += ["## Connections by Relation", ""]
+    for rel, targets in sorted(by_relation.items()):
+        lines.append(f"### {rel}")
+        for t in targets[:20]:
+            lines.append(f"- {t}")
+        lines.append("")
+
+    lines += ["---", "", "*Part of the graphify knowledge wiki. See [[index]] to navigate.*"]
+    return "\n".join(lines)
+
+
+def _index_md(
+    communities: dict[int, list[str]],
+    labels: dict[int, str],
+    god_nodes_data: list[dict],
+    total_nodes: int,
+    total_edges: int,
+) -> str:
+    lines: list[str] = [
+        "# Knowledge Graph Index",
+        "",
+        "> Auto-generated by graphify. Start here — read community articles for context, then drill into god nodes for detail.",
+        "",
+        f"**{total_nodes} nodes · {total_edges} edges · {len(communities)} communities**",
+        "",
+        "---",
+        "",
+        "## Communities",
+        "(sorted by size, largest first)",
+        "",
+    ]
+
+    for cid, nodes in sorted(communities.items(), key=lambda x: -len(x[1])):
+        label = labels.get(cid, f"Community {cid}")
+        lines.append(f"- [[{label}]] — {len(nodes)} nodes")
+    lines.append("")
+
+    if god_nodes_data:
+        lines += ["## God Nodes", "(most connected concepts — the load-bearing abstractions)", ""]
+        for node in god_nodes_data:
+            lines.append(f"- [[{node['label']}]] — {node['edges']} connections")
+        lines.append("")
+
+    lines += [
+        "---",
+        "",
+        "*Generated by [graphify](https://github.com/safishamsi/graphify)*",
+    ]
+    return "\n".join(lines)
+
+
+def to_wiki(
+    G: nx.Graph,
+    communities: dict[int, list[str]],
+    output_dir: str | Path,
+    community_labels: dict[int, str] | None = None,
+    cohesion: dict[int, float] | None = None,
+    god_nodes_data: list[dict] | None = None,
+) -> int:
+    """Generate a Wikipedia-style wiki from the graph.
+
+    Writes:
+      - index.md            — agent entry point, catalog of all articles
+      - <CommunityName>.md  — one article per community
+      - <GodNodeLabel>.md   — one article per god node
+
+    Returns the number of articles written (excluding index.md).
+    """
+    out = Path(output_dir)
+    out.mkdir(parents=True, exist_ok=True)
+
+    labels = community_labels or {cid: f"Community {cid}" for cid in communities}
+    cohesion = cohesion or {}
+    god_nodes_data = god_nodes_data or []
+
+    count = 0
+
+    # Community articles
+    for cid, nodes in communities.items():
+        label = labels.get(cid, f"Community {cid}")
+        article = _community_article(G, cid, nodes, label, labels, cohesion.get(cid))
+        (out / f"{_safe_filename(label)}.md").write_text(article)
+        count += 1
+
+    # God node articles
+    for node_data in god_nodes_data:
+        nid = node_data.get("id")
+        if nid and nid in G:
+            article = _god_node_article(G, nid, labels)
+            (out / f"{_safe_filename(node_data['label'])}.md").write_text(article)
+            count += 1
+
+    # Index
+    (out / "index.md").write_text(
+        _index_md(communities, labels, god_nodes_data, G.number_of_nodes(), G.number_of_edges())
+    )
+
+    return count
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

 [project]
 name = "graphifyy"
-version = "0.1.6"
+version = "0.1.7"
 description = "Claude Code skill - turn any folder of code, docs, papers, images, or tweets into a queryable knowledge graph"
 readme = "README.md"
 license = { text = "MIT" }
@@ -20,6 +20,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti
 /graphify <path> --html                               # (HTML is generated by default - this flag is a no-op)
 /graphify <path> --svg                                # also export graph.svg (embeds in Notion, GitHub)
 /graphify <path> --graphml                            # export graph.graphml (Gephi, yEd)
+/graphify <path> --wiki                               # export agent-crawlable wiki (index.md + article per community + god nodes)
 /graphify <path> --neo4j                              # generate graphify-out/cypher.txt for Neo4j
 /graphify <path> --neo4j-push bolt://localhost:7687   # push directly to Neo4j
 /graphify <path> --mcp                                # start MCP stdio server for agent access
@@ -522,7 +523,42 @@ print('graph.graphml written - open in Gephi, yEd, or any GraphML tool')
 "
 ```

-### Step 7d - MCP server (only if --mcp flag)
+### Step 7d - Wiki export (only if --wiki flag)
+
+Generates a Wikipedia-style markdown wiki: one article per community, one per god node, plus an `index.md` entry point for agents to start from. Inspired by the Farzapedia pattern — structure the knowledge so an agent can navigate it like a file system it understands.
+
+```bash
+python3 -c "
+import json
+from graphify.build import build_from_json
+from graphify.analyze import god_nodes
+from graphify.wiki import to_wiki
+from pathlib import Path
+
+extraction = json.loads(Path('.graphify_extract.json').read_text())
+analysis   = json.loads(Path('.graphify_analysis.json').read_text())
+labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {}
+
+G = build_from_json(extraction)
+communities = {int(k): v for k, v in analysis['communities'].items()}
+cohesion    = {int(k): v for k, v in analysis['cohesion'].items()}
+labels      = {int(k): v for k, v in labels_raw.items()}
+gods        = god_nodes(G, top_n=20)
+
+n = to_wiki(G, communities, 'graphify-out/wiki', community_labels=labels or None, cohesion=cohesion, god_nodes_data=gods)
+print(f'Wiki: {n} articles written to graphify-out/wiki/')
+print('Start at graphify-out/wiki/index.md')
+"
+```
+
+The wiki contains:
+- `index.md` — catalog of all communities and god nodes; agent entry point
+- `<CommunityName>.md` — key concepts, cross-community links, source files, audit trail
+- `<GodNodeLabel>.md` — all connections grouped by relation type, community membership
+
+To use with an agent: point it at `index.md` and tell it to navigate the wiki to answer questions about the corpus. Works with Claude Code, Claude Desktop, or any agent that can read markdown files.
+
+### Step 7e - MCP server (only if --mcp flag)

 ```bash
 python3 -m graphify.serve graphify-out/graph.json
@@ -1138,10 +1174,19 @@ Then answer using graph data:

 Example — finding all verbs (action concepts) in a codebase:
 ```python
-# Functions and methods are the verbs of code
-verbs = [(d["label"], d.get("source_file", "")) for _, d in G.nodes(data=True)
-         if d.get("file_type") == "code" and any(k in d.get("label", "").lower()
-         for k in ["()", "fn ", "def ", "func"])]
+from collections import Counter
+
+# Node labels are plain names like "run", "render", "resolve" — no "def"/"fn" prefix
+# Extract the first word of each function label (e.g. "load_graph" → "load")
+verb_counts = Counter()
+for _, d in G.nodes(data=True):
+    if d.get("file_type") == "code":
+        first_word = d.get("label", "").split("_")[0].split(".")[0].lower()
+        if first_word and first_word.isalpha():
+            verb_counts[first_word] += 1
+
+for verb, count in verb_counts.most_common(20):
+    print(f"{count:>4}x  {verb}")
 ```

 **The only exception:** if the user explicitly asks you to look at a raw file (e.g., "show me the contents of X"), you may read that specific file. But for any analytical question, use the graph.
@@ -0,0 +1,125 @@
+"""Tests for graphify.wiki — Wikipedia-style article generation."""
+import pytest
+from pathlib import Path
+import networkx as nx
+from graphify.wiki import to_wiki, _index_md, _community_article, _god_node_article
+
+
+def _make_graph():
+    G = nx.Graph()
+    G.add_node("n1", label="parse", file_type="code", source_file="parser.py", community=0)
+    G.add_node("n2", label="validate", file_type="code", source_file="parser.py", community=0)
+    G.add_node("n3", label="render", file_type="code", source_file="renderer.py", community=1)
+    G.add_node("n4", label="stream", file_type="code", source_file="renderer.py", community=1)
+    G.add_edge("n1", "n2", relation="calls", confidence="EXTRACTED", weight=1.0)
+    G.add_edge("n1", "n3", relation="references", confidence="INFERRED", weight=1.0)
+    G.add_edge("n3", "n4", relation="calls", confidence="EXTRACTED", weight=1.0)
+    return G
+
+
+COMMUNITIES = {0: ["n1", "n2"], 1: ["n3", "n4"]}
+LABELS = {0: "Parsing Layer", 1: "Rendering Layer"}
+COHESION = {0: 0.85, 1: 0.72}
+GOD_NODES = [{"id": "n1", "label": "parse", "edges": 2}]
+
+
+def test_to_wiki_writes_index(tmp_path):
+    G = _make_graph()
+    n = to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION, god_nodes_data=GOD_NODES)
+    assert (tmp_path / "index.md").exists()
+
+
+def test_to_wiki_returns_article_count(tmp_path):
+    G = _make_graph()
+    # 2 communities + 1 god node = 3
+    n = to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION, god_nodes_data=GOD_NODES)
+    assert n == 3
+
+
+def test_to_wiki_community_articles_created(tmp_path):
+    G = _make_graph()
+    to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS)
+    assert (tmp_path / "Parsing_Layer.md").exists()
+    assert (tmp_path / "Rendering_Layer.md").exists()
+
+
+def test_to_wiki_god_node_article_created(tmp_path):
+    G = _make_graph()
+    to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES)
+    assert (tmp_path / "parse.md").exists()
+
+
+def test_index_links_all_communities(tmp_path):
+    G = _make_graph()
+    to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS)
+    index = (tmp_path / "index.md").read_text()
+    assert "[[Parsing Layer]]" in index
+    assert "[[Rendering Layer]]" in index
+
+
+def test_index_lists_god_nodes(tmp_path):
+    G = _make_graph()
+    to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES)
+    index = (tmp_path / "index.md").read_text()
+    assert "[[parse]]" in index
+    assert "2 connections" in index
+
+
+def test_community_article_has_cross_links(tmp_path):
+    G = _make_graph()
+    to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS)
+    parsing = (tmp_path / "Parsing_Layer.md").read_text()
+    # n1 (parsing) references n3 (rendering) → cross-community link
+    assert "[[Rendering Layer]]" in parsing
+
+
+def test_community_article_shows_cohesion(tmp_path):
+    G = _make_graph()
+    to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION)
+    parsing = (tmp_path / "Parsing_Layer.md").read_text()
+    assert "cohesion 0.85" in parsing
+
+
+def test_community_article_has_audit_trail(tmp_path):
+    G = _make_graph()
+    to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS)
+    parsing = (tmp_path / "Parsing_Layer.md").read_text()
+    assert "EXTRACTED" in parsing
+    assert "INFERRED" in parsing
+
+
+def test_god_node_article_has_connections(tmp_path):
+    G = _make_graph()
+    to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES)
+    article = (tmp_path / "parse.md").read_text()
+    assert "[[validate]]" in article or "[[render]]" in article
+
+
+def test_god_node_article_links_community(tmp_path):
+    G = _make_graph()
+    to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES)
+    article = (tmp_path / "parse.md").read_text()
+    assert "[[Parsing Layer]]" in article
+
+
+def test_to_wiki_skips_missing_god_node_ids(tmp_path):
+    """God node with bad ID should not crash."""
+    G = _make_graph()
+    bad_gods = [{"id": "nonexistent", "label": "ghost", "edges": 99}]
+    n = to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=bad_gods)
+    # 2 communities + 0 god nodes (nonexistent skipped) = 2
+    assert n == 2
+
+
+def test_to_wiki_no_labels_uses_fallback(tmp_path):
+    G = _make_graph()
+    to_wiki(G, COMMUNITIES, tmp_path)  # no labels
+    assert (tmp_path / "Community_0.md").exists()
+    assert (tmp_path / "Community_1.md").exists()
+
+
+def test_article_navigation_footer(tmp_path):
+    G = _make_graph()
+    to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS)
+    article = (tmp_path / "Parsing_Layer.md").read_text()
+    assert "[[index]]" in article