diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ff41c18..a50669ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 0.1.7 (2026-04-05) + +- Add: `--wiki` flag — generates Wikipedia-style agent-crawlable wiki from the graph (index.md + community articles + god node articles) +- Add: `graphify/wiki.py` module with `to_wiki()` — cross-community wikilinks, cohesion scores, audit trail, navigation footer +- Add: 14 wiki tests (245 total) +- Fix: follow-up question example code now correctly splits node labels by `_` to extract verb prefixes (previous version used `def`/`fn` prefix matching which always returned zero results) + ## 0.1.6 (2026-04-05) - Fix: follow-up questions after pipeline now answered from graph.json, not by re-exploring the directory (was 25 tool calls / 1m30s; now instant) diff --git a/graphify/__init__.py b/graphify/__init__.py index 72fdb9b8..e34c938e 100644 --- a/graphify/__init__.py +++ b/graphify/__init__.py @@ -18,6 +18,7 @@ def __getattr__(name): "to_html": ("graphify.export", "to_html"), "to_svg": ("graphify.export", "to_svg"), "to_canvas": ("graphify.export", "to_canvas"), + "to_wiki": ("graphify.wiki", "to_wiki"), } if name in _map: import importlib diff --git a/graphify/wiki.py b/graphify/wiki.py new file mode 100644 index 00000000..20f83a16 --- /dev/null +++ b/graphify/wiki.py @@ -0,0 +1,211 @@ +# Wiki export - Wikipedia-style markdown articles from the knowledge graph +# Generates an agent-crawlable wiki: index.md + one article per community + god node articles +from __future__ import annotations +from collections import Counter +from pathlib import Path +import networkx as nx + + +def _safe_filename(name: str) -> str: + return name.replace("/", "-").replace(" ", "_").replace(":", "-") + + +def _cross_community_links(G: nx.Graph, nodes: list[str], own_cid: int, labels: dict[int, str]) -> list[tuple[str, int]]: + """Return (community_label, edge_count) pairs for cross-community connections, sorted descending.""" + counts: dict[str, int] = Counter() + for nid in nodes: + for neighbor in G.neighbors(nid): + nd = G.nodes[neighbor] + ncid = nd.get("community") + if ncid is not None and ncid != own_cid: + counts[labels.get(ncid, f"Community {ncid}")] += 1 + return sorted(counts.items(), key=lambda x: -x[1]) + + +def _community_article( + G: nx.Graph, + cid: int, + nodes: list[str], + label: str, + labels: dict[int, str], + cohesion: float | None, +) -> str: + top_nodes = sorted(nodes, key=lambda n: G.degree(n), reverse=True)[:25] + cross = _cross_community_links(G, nodes, cid, labels) + + # Edge confidence breakdown + conf_counts: Counter = Counter() + for nid in nodes: + for neighbor in G.neighbors(nid): + ed = G.edges[nid, neighbor] + conf_counts[ed.get("confidence", "EXTRACTED")] += 1 + total_edges = sum(conf_counts.values()) or 1 + + sources = sorted({G.nodes[n].get("source_file", "") for n in nodes} - {""}) + + lines: list[str] = [] + lines += [f"# {label}", ""] + + meta_parts = [f"{len(nodes)} nodes"] + if cohesion is not None: + meta_parts.append(f"cohesion {cohesion:.2f}") + lines += [f"> {' · '.join(meta_parts)}", ""] + + lines += ["## Key Concepts", ""] + for nid in top_nodes: + d = G.nodes[nid] + node_label = d.get("label", nid) + src = d.get("source_file", "") + degree = G.degree(nid) + src_str = f" — `{src}`" if src else "" + lines.append(f"- **{node_label}** ({degree} connections){src_str}") + lines.append("") + + lines += ["## Relationships", ""] + if cross: + for other_label, count in cross[:12]: + lines.append(f"- [[{other_label}]] ({count} shared connections)") + else: + lines.append("- No strong cross-community connections detected") + lines.append("") + + if sources: + lines += ["## Source Files", ""] + for src in sources[:20]: + lines.append(f"- `{src}`") + lines.append("") + + lines += ["## Audit Trail", ""] + for conf in ("EXTRACTED", "INFERRED", "AMBIGUOUS"): + n = conf_counts.get(conf, 0) + pct = round(n / total_edges * 100) + lines.append(f"- {conf}: {n} ({pct}%)") + lines.append("") + + lines += ["---", "", "*Part of the graphify knowledge wiki. See [[index]] to navigate.*"] + return "\n".join(lines) + + +def _god_node_article(G: nx.Graph, nid: str, labels: dict[int, str]) -> str: + d = G.nodes[nid] + node_label = d.get("label", nid) + src = d.get("source_file", "") + cid = d.get("community") + community_name = labels.get(cid, f"Community {cid}") if cid is not None else None + + lines: list[str] = [] + lines += [f"# {node_label}", ""] + lines += [f"> God node · {G.degree(nid)} connections · `{src}`", ""] + + if community_name: + lines += [f"**Community:** [[{community_name}]]", ""] + + # Group neighbors by relation type + by_relation: dict[str, list[str]] = {} + for neighbor in sorted(G.neighbors(nid), key=lambda n: G.degree(n), reverse=True): + nd = G.nodes[neighbor] + ed = G.edges[nid, neighbor] + rel = ed.get("relation", "related") + neighbor_label = nd.get("label", neighbor) + conf = ed.get("confidence", "") + conf_str = f" `{conf}`" if conf else "" + by_relation.setdefault(rel, []).append(f"[[{neighbor_label}]]{conf_str}") + + lines += ["## Connections by Relation", ""] + for rel, targets in sorted(by_relation.items()): + lines.append(f"### {rel}") + for t in targets[:20]: + lines.append(f"- {t}") + lines.append("") + + lines += ["---", "", "*Part of the graphify knowledge wiki. See [[index]] to navigate.*"] + return "\n".join(lines) + + +def _index_md( + communities: dict[int, list[str]], + labels: dict[int, str], + god_nodes_data: list[dict], + total_nodes: int, + total_edges: int, +) -> str: + lines: list[str] = [ + "# Knowledge Graph Index", + "", + "> Auto-generated by graphify. Start here — read community articles for context, then drill into god nodes for detail.", + "", + f"**{total_nodes} nodes · {total_edges} edges · {len(communities)} communities**", + "", + "---", + "", + "## Communities", + "(sorted by size, largest first)", + "", + ] + + for cid, nodes in sorted(communities.items(), key=lambda x: -len(x[1])): + label = labels.get(cid, f"Community {cid}") + lines.append(f"- [[{label}]] — {len(nodes)} nodes") + lines.append("") + + if god_nodes_data: + lines += ["## God Nodes", "(most connected concepts — the load-bearing abstractions)", ""] + for node in god_nodes_data: + lines.append(f"- [[{node['label']}]] — {node['edges']} connections") + lines.append("") + + lines += [ + "---", + "", + "*Generated by [graphify](https://github.com/safishamsi/graphify)*", + ] + return "\n".join(lines) + + +def to_wiki( + G: nx.Graph, + communities: dict[int, list[str]], + output_dir: str | Path, + community_labels: dict[int, str] | None = None, + cohesion: dict[int, float] | None = None, + god_nodes_data: list[dict] | None = None, +) -> int: + """Generate a Wikipedia-style wiki from the graph. + + Writes: + - index.md — agent entry point, catalog of all articles + - .md — one article per community + - .md — one article per god node + + Returns the number of articles written (excluding index.md). + """ + out = Path(output_dir) + out.mkdir(parents=True, exist_ok=True) + + labels = community_labels or {cid: f"Community {cid}" for cid in communities} + cohesion = cohesion or {} + god_nodes_data = god_nodes_data or [] + + count = 0 + + # Community articles + for cid, nodes in communities.items(): + label = labels.get(cid, f"Community {cid}") + article = _community_article(G, cid, nodes, label, labels, cohesion.get(cid)) + (out / f"{_safe_filename(label)}.md").write_text(article) + count += 1 + + # God node articles + for node_data in god_nodes_data: + nid = node_data.get("id") + if nid and nid in G: + article = _god_node_article(G, nid, labels) + (out / f"{_safe_filename(node_data['label'])}.md").write_text(article) + count += 1 + + # Index + (out / "index.md").write_text( + _index_md(communities, labels, god_nodes_data, G.number_of_nodes(), G.number_of_edges()) + ) + + return count diff --git a/pyproject.toml b/pyproject.toml index 97460a9e..2a914112 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.1.6" +version = "0.1.7" description = "Claude Code skill - turn any folder of code, docs, papers, images, or tweets into a queryable knowledge graph" readme = "README.md" license = { text = "MIT" } diff --git a/skills/graphify/skill.md b/skills/graphify/skill.md index dc13efe6..3a6d63f5 100644 --- a/skills/graphify/skill.md +++ b/skills/graphify/skill.md @@ -20,6 +20,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --html # (HTML is generated by default - this flag is a no-op) /graphify --svg # also export graph.svg (embeds in Notion, GitHub) /graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --wiki # export agent-crawlable wiki (index.md + article per community + god nodes) /graphify --neo4j # generate graphify-out/cypher.txt for Neo4j /graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j /graphify --mcp # start MCP stdio server for agent access @@ -522,7 +523,42 @@ print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') " ``` -### Step 7d - MCP server (only if --mcp flag) +### Step 7d - Wiki export (only if --wiki flag) + +Generates a Wikipedia-style markdown wiki: one article per community, one per god node, plus an `index.md` entry point for agents to start from. Inspired by the Farzapedia pattern — structure the knowledge so an agent can navigate it like a file system it understands. + +```bash +python3 -c " +import json +from graphify.build import build_from_json +from graphify.analyze import god_nodes +from graphify.wiki import to_wiki +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +labels = {int(k): v for k, v in labels_raw.items()} +gods = god_nodes(G, top_n=20) + +n = to_wiki(G, communities, 'graphify-out/wiki', community_labels=labels or None, cohesion=cohesion, god_nodes_data=gods) +print(f'Wiki: {n} articles written to graphify-out/wiki/') +print('Start at graphify-out/wiki/index.md') +" +``` + +The wiki contains: +- `index.md` — catalog of all communities and god nodes; agent entry point +- `.md` — key concepts, cross-community links, source files, audit trail +- `.md` — all connections grouped by relation type, community membership + +To use with an agent: point it at `index.md` and tell it to navigate the wiki to answer questions about the corpus. Works with Claude Code, Claude Desktop, or any agent that can read markdown files. + +### Step 7e - MCP server (only if --mcp flag) ```bash python3 -m graphify.serve graphify-out/graph.json @@ -1138,10 +1174,19 @@ Then answer using graph data: Example — finding all verbs (action concepts) in a codebase: ```python -# Functions and methods are the verbs of code -verbs = [(d["label"], d.get("source_file", "")) for _, d in G.nodes(data=True) - if d.get("file_type") == "code" and any(k in d.get("label", "").lower() - for k in ["()", "fn ", "def ", "func"])] +from collections import Counter + +# Node labels are plain names like "run", "render", "resolve" — no "def"/"fn" prefix +# Extract the first word of each function label (e.g. "load_graph" → "load") +verb_counts = Counter() +for _, d in G.nodes(data=True): + if d.get("file_type") == "code": + first_word = d.get("label", "").split("_")[0].split(".")[0].lower() + if first_word and first_word.isalpha(): + verb_counts[first_word] += 1 + +for verb, count in verb_counts.most_common(20): + print(f"{count:>4}x {verb}") ``` **The only exception:** if the user explicitly asks you to look at a raw file (e.g., "show me the contents of X"), you may read that specific file. But for any analytical question, use the graph. diff --git a/tests/test_wiki.py b/tests/test_wiki.py new file mode 100644 index 00000000..686d0d69 --- /dev/null +++ b/tests/test_wiki.py @@ -0,0 +1,125 @@ +"""Tests for graphify.wiki — Wikipedia-style article generation.""" +import pytest +from pathlib import Path +import networkx as nx +from graphify.wiki import to_wiki, _index_md, _community_article, _god_node_article + + +def _make_graph(): + G = nx.Graph() + G.add_node("n1", label="parse", file_type="code", source_file="parser.py", community=0) + G.add_node("n2", label="validate", file_type="code", source_file="parser.py", community=0) + G.add_node("n3", label="render", file_type="code", source_file="renderer.py", community=1) + G.add_node("n4", label="stream", file_type="code", source_file="renderer.py", community=1) + G.add_edge("n1", "n2", relation="calls", confidence="EXTRACTED", weight=1.0) + G.add_edge("n1", "n3", relation="references", confidence="INFERRED", weight=1.0) + G.add_edge("n3", "n4", relation="calls", confidence="EXTRACTED", weight=1.0) + return G + + +COMMUNITIES = {0: ["n1", "n2"], 1: ["n3", "n4"]} +LABELS = {0: "Parsing Layer", 1: "Rendering Layer"} +COHESION = {0: 0.85, 1: 0.72} +GOD_NODES = [{"id": "n1", "label": "parse", "edges": 2}] + + +def test_to_wiki_writes_index(tmp_path): + G = _make_graph() + n = to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION, god_nodes_data=GOD_NODES) + assert (tmp_path / "index.md").exists() + + +def test_to_wiki_returns_article_count(tmp_path): + G = _make_graph() + # 2 communities + 1 god node = 3 + n = to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION, god_nodes_data=GOD_NODES) + assert n == 3 + + +def test_to_wiki_community_articles_created(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) + assert (tmp_path / "Parsing_Layer.md").exists() + assert (tmp_path / "Rendering_Layer.md").exists() + + +def test_to_wiki_god_node_article_created(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES) + assert (tmp_path / "parse.md").exists() + + +def test_index_links_all_communities(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) + index = (tmp_path / "index.md").read_text() + assert "[[Parsing Layer]]" in index + assert "[[Rendering Layer]]" in index + + +def test_index_lists_god_nodes(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES) + index = (tmp_path / "index.md").read_text() + assert "[[parse]]" in index + assert "2 connections" in index + + +def test_community_article_has_cross_links(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) + parsing = (tmp_path / "Parsing_Layer.md").read_text() + # n1 (parsing) references n3 (rendering) → cross-community link + assert "[[Rendering Layer]]" in parsing + + +def test_community_article_shows_cohesion(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION) + parsing = (tmp_path / "Parsing_Layer.md").read_text() + assert "cohesion 0.85" in parsing + + +def test_community_article_has_audit_trail(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) + parsing = (tmp_path / "Parsing_Layer.md").read_text() + assert "EXTRACTED" in parsing + assert "INFERRED" in parsing + + +def test_god_node_article_has_connections(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES) + article = (tmp_path / "parse.md").read_text() + assert "[[validate]]" in article or "[[render]]" in article + + +def test_god_node_article_links_community(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES) + article = (tmp_path / "parse.md").read_text() + assert "[[Parsing Layer]]" in article + + +def test_to_wiki_skips_missing_god_node_ids(tmp_path): + """God node with bad ID should not crash.""" + G = _make_graph() + bad_gods = [{"id": "nonexistent", "label": "ghost", "edges": 99}] + n = to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=bad_gods) + # 2 communities + 0 god nodes (nonexistent skipped) = 2 + assert n == 2 + + +def test_to_wiki_no_labels_uses_fallback(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path) # no labels + assert (tmp_path / "Community_0.md").exists() + assert (tmp_path / "Community_1.md").exists() + + +def test_article_navigation_footer(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) + article = (tmp_path / "Parsing_Layer.md").read_text() + assert "[[index]]" in article