diff --git a/graphify/analyze.py b/graphify/analyze.py index 995b5fb7..cf534496 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -3,6 +3,11 @@ from __future__ import annotations import networkx as nx +def _node_community_map(communities: dict[int, list[str]]) -> dict[str, int]: + """Invert communities dict: node_id -> community_id.""" + return {n: cid for cid, nodes in communities.items() for n in nodes} + + def _is_file_node(G: nx.Graph, node_id: str) -> bool: """ Return True if this node is a file-level hub node (e.g. 'client', 'models') @@ -187,7 +192,7 @@ def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: Each result includes a 'why' field explaining what makes it non-obvious. """ - node_community = {n: cid for cid, nodes in communities.items() for n in nodes} + node_community = _node_community_map(communities) candidates = [] for u, v, data in G.edges(data=True): @@ -266,7 +271,7 @@ def _cross_community_surprises( return result # Build node → community map - node_community = {n: cid for cid, nodes in communities.items() for n in nodes} + node_community = _node_community_map(communities) surprises = [] for u, v, data in G.edges(data=True): @@ -325,7 +330,7 @@ def suggest_questions( Each question has a 'type', 'question', and 'why' field. """ questions = [] - node_community = {n: cid for cid, nodes in communities.items() for n in nodes} + node_community = _node_community_map(communities) # 1. AMBIGUOUS edges → unresolved relationship questions for u, v, data in G.edges(data=True): diff --git a/graphify/cache.py b/graphify/cache.py index 99db860d..0bde3bb7 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -3,7 +3,6 @@ from __future__ import annotations import hashlib import json -import shutil from pathlib import Path diff --git a/graphify/detect.py b/graphify/detect.py index 7c623f3d..3c740688 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -1,6 +1,7 @@ # file discovery, type classification, and corpus health checks from __future__ import annotations import json +import os import re from enum import Enum from pathlib import Path @@ -155,7 +156,6 @@ def detect(root: Path) -> dict: for scan_root in scan_paths: in_memory_tree = memory_dir.exists() and str(scan_root).startswith(str(memory_dir)) - import os for dirpath, dirnames, filenames in os.walk(scan_root, followlinks=False): dp = Path(dirpath) if not in_memory_tree: diff --git a/graphify/export.py b/graphify/export.py index 9035f3dc..14306328 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -17,8 +17,186 @@ COMMUNITY_COLORS = [ MAX_NODES_FOR_VIZ = 5_000 +def _node_community_map(communities: dict[int, list[str]]) -> dict[str, int]: + """Invert communities dict: node_id -> community_id.""" + return {n: cid for cid, nodes in communities.items() for n in nodes} + + +def _html_styles() -> str: + return """""" + + +def _html_script(nodes_json: str, edges_json: str, legend_json: str) -> str: + return f"""""" + + def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str) -> None: - node_community = {n: cid for cid, nodes in communities.items() for n in nodes} + node_community = _node_community_map(communities) data = json_graph.node_link_data(G, edges="links") for node in data["nodes"]: node["community"] = node_community.get(node["id"]) @@ -62,7 +240,7 @@ def to_html( f"Use --no-viz or reduce input size." ) - node_community = {n: cid for cid, nodes in communities.items() for n in nodes} + node_community = _node_community_map(communities) degree = dict(G.degree()) max_deg = max(degree.values()) if degree else 1 @@ -118,6 +296,7 @@ def to_html( edges_json = json.dumps(vis_edges) legend_json = json.dumps(legend_data) title = sanitize_label(str(output_path)) + stats = f"{G.number_of_nodes()} nodes · {G.number_of_edges()} edges · {len(communities)} communities" html = f""" @@ -125,36 +304,7 @@ def to_html( graphify - {title} - +{_html_styles()}
@@ -171,160 +321,9 @@ def to_html(

Communities

-
- {G.number_of_nodes()} nodes · {G.number_of_edges()} edges · {len(communities)} communities -
+
{stats}
- - +{_html_script(nodes_json, edges_json, legend_json)} """ @@ -353,7 +352,7 @@ def to_obsidian( out = Path(output_dir) out.mkdir(parents=True, exist_ok=True) - node_community = {n: cid for cid, nodes in communities.items() for n in nodes} + node_community = _node_community_map(communities) # Map node_id → safe filename so wikilinks stay consistent. # Deduplicate: if two nodes produce the same filename, append a numeric suffix. @@ -755,10 +754,7 @@ def push_to_neo4j( "neo4j driver not installed. Run: pip install neo4j" ) from e - node_community = ( - {n: cid for cid, nodes in communities.items() for n in nodes} - if communities else {} - ) + node_community = _node_community_map(communities) if communities else {} def _safe_rel(relation: str) -> str: return re.sub(r"[^A-Z0-9_]", "_", relation.upper().replace(" ", "_").replace("-", "_")) or "RELATED_TO" @@ -809,7 +805,7 @@ def to_graphml( Edge confidence (EXTRACTED/INFERRED/AMBIGUOUS) is preserved as an edge attribute. """ H = G.copy() - node_community = {n: cid for cid, nodes in communities.items() for n in nodes} + node_community = _node_community_map(communities) for node_id in H.nodes(): H.nodes[node_id]["community"] = node_community.get(node_id, -1) nx.write_graphml(H, output_path) @@ -837,7 +833,7 @@ def to_svg( except ImportError as e: raise ImportError("matplotlib not installed. Run: pip install matplotlib") from e - node_community = {n: cid for cid, nodes in communities.items() for n in nodes} + node_community = _node_community_map(communities) fig, ax = plt.subplots(figsize=figsize, facecolor="#1a1a2e") ax.set_facecolor("#1a1a2e") diff --git a/graphify/ingest.py b/graphify/ingest.py index 70be4498..7441e867 100644 --- a/graphify/ingest.py +++ b/graphify/ingest.py @@ -2,7 +2,6 @@ from __future__ import annotations import json import re -import sys import urllib.error import urllib.parse from datetime import datetime, timezone diff --git a/graphify/serve.py b/graphify/serve.py index cc1a398f..0d086d54 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -93,6 +93,13 @@ def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_bu return output +def _find_node(G: nx.Graph, label: str) -> list[str]: + """Return node IDs whose label or ID matches the search term (case-insensitive).""" + term = label.lower() + return [nid for nid, d in G.nodes(data=True) + if term in d.get("label", "").lower() or term == nid.lower()] + + def serve(graph_path: str = "graphify-out/graph.json") -> None: """Start the MCP server. Requires pip install mcp.""" try: @@ -130,9 +137,7 @@ def serve(graph_path: str = "graphify-out/graph.json") -> None: description="Get full details for a specific node by label or ID.", inputSchema={ "type": "object", - "properties": { - "label": {"type": "string", "description": "Node label or ID to look up"}, - }, + "properties": {"label": {"type": "string", "description": "Node label or ID to look up"}}, "required": ["label"], }, ), @@ -150,24 +155,17 @@ def serve(graph_path: str = "graphify-out/graph.json") -> None: ), types.Tool( name="get_community", - description="Get all nodes in a community by community ID or label.", + description="Get all nodes in a community by community ID.", inputSchema={ "type": "object", - "properties": { - "community_id": {"type": "integer", "description": "Community ID (0-indexed by size)"}, - }, + "properties": {"community_id": {"type": "integer", "description": "Community ID (0-indexed by size)"}}, "required": ["community_id"], }, ), types.Tool( name="god_nodes", description="Return the most connected nodes - the core abstractions of the knowledge graph.", - inputSchema={ - "type": "object", - "properties": { - "top_n": {"type": "integer", "default": 10}, - }, - }, + inputSchema={"type": "object", "properties": {"top_n": {"type": "integer", "default": 10}}}, ), types.Tool( name="graph_stats", @@ -189,130 +187,126 @@ def serve(graph_path: str = "graphify-out/graph.json") -> None: ), ] + def _tool_query_graph(arguments: dict) -> str: + question = arguments["question"] + mode = arguments.get("mode", "bfs") + depth = min(int(arguments.get("depth", 3)), 6) + budget = int(arguments.get("token_budget", 2000)) + terms = [t.lower() for t in question.split() if len(t) > 2] + scored = _score_nodes(G, terms) + start_nodes = [nid for _, nid in scored[:3]] + if not start_nodes: + return "No matching nodes found." + nodes, edges = _dfs(G, start_nodes, depth) if mode == "dfs" else _bfs(G, start_nodes, depth) + header = f"Traversal: {mode.upper()} depth={depth} | Start: {[G.nodes[n].get('label', n) for n in start_nodes]} | {len(nodes)} nodes found\n\n" + return header + _subgraph_to_text(G, nodes, edges, budget) + + def _tool_get_node(arguments: dict) -> str: + label = arguments["label"].lower() + matches = [(nid, d) for nid, d in G.nodes(data=True) + if label in d.get("label", "").lower() or label == nid.lower()] + if not matches: + return f"No node matching '{label}' found." + nid, d = matches[0] + return "\n".join([ + f"Node: {d.get('label', nid)}", + f" ID: {nid}", + f" Source: {d.get('source_file', '')} {d.get('source_location', '')}", + f" Type: {d.get('file_type', '')}", + f" Community: {d.get('community', '')}", + f" Degree: {G.degree(nid)}", + ]) + + def _tool_get_neighbors(arguments: dict) -> str: + label = arguments["label"].lower() + rel_filter = arguments.get("relation_filter", "").lower() + matches = _find_node(G, label) + if not matches: + return f"No node matching '{label}' found." + nid = matches[0] + lines = [f"Neighbors of {G.nodes[nid].get('label', nid)}:"] + for neighbor in G.neighbors(nid): + d = G.edges[nid, neighbor] + rel = d.get("relation", "") + if rel_filter and rel_filter not in rel.lower(): + continue + lines.append(f" --> {G.nodes[neighbor].get('label', neighbor)} [{rel}] [{d.get('confidence', '')}]") + return "\n".join(lines) + + def _tool_get_community(arguments: dict) -> str: + cid = int(arguments["community_id"]) + nodes = communities.get(cid, []) + if not nodes: + return f"Community {cid} not found." + lines = [f"Community {cid} ({len(nodes)} nodes):"] + for n in nodes: + d = G.nodes[n] + lines.append(f" {d.get('label', n)} [{d.get('source_file', '')}]") + return "\n".join(lines) + + def _tool_god_nodes(arguments: dict) -> str: + from .analyze import god_nodes as _god_nodes + nodes = _god_nodes(G, top_n=int(arguments.get("top_n", 10))) + lines = ["God nodes (most connected):"] + lines += [f" {i}. {n['label']} - {n['edges']} edges" for i, n in enumerate(nodes, 1)] + return "\n".join(lines) + + def _tool_graph_stats(_: dict) -> str: + confs = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)] + total = len(confs) or 1 + return ( + f"Nodes: {G.number_of_nodes()}\n" + f"Edges: {G.number_of_edges()}\n" + f"Communities: {len(communities)}\n" + f"EXTRACTED: {round(confs.count('EXTRACTED')/total*100)}%\n" + f"INFERRED: {round(confs.count('INFERRED')/total*100)}%\n" + f"AMBIGUOUS: {round(confs.count('AMBIGUOUS')/total*100)}%\n" + ) + + def _tool_shortest_path(arguments: dict) -> str: + src_scored = _score_nodes(G, [t.lower() for t in arguments["source"].split()]) + tgt_scored = _score_nodes(G, [t.lower() for t in arguments["target"].split()]) + if not src_scored: + return f"No node matching source '{arguments['source']}' found." + if not tgt_scored: + return f"No node matching target '{arguments['target']}' found." + src_nid, tgt_nid = src_scored[0][1], tgt_scored[0][1] + max_hops = int(arguments.get("max_hops", 8)) + try: + path_nodes = nx.shortest_path(G, src_nid, tgt_nid) + except (nx.NetworkXNoPath, nx.NodeNotFound): + return f"No path found between '{G.nodes[src_nid].get('label', src_nid)}' and '{G.nodes[tgt_nid].get('label', tgt_nid)}'." + hops = len(path_nodes) - 1 + if hops > max_hops: + return f"Path exceeds max_hops={max_hops} ({hops} hops found)." + segments = [] + for i in range(len(path_nodes) - 1): + u, v = path_nodes[i], path_nodes[i + 1] + edata = G.edges[u, v] + rel = edata.get("relation", "") + conf = edata.get("confidence", "") + conf_str = f" [{conf}]" if conf else "" + if i == 0: + segments.append(G.nodes[u].get("label", u)) + segments.append(f"--{rel}{conf_str}--> {G.nodes[v].get('label', v)}") + return f"Shortest path ({hops} hops):\n " + " ".join(segments) + + _handlers = { + "query_graph": _tool_query_graph, + "get_node": _tool_get_node, + "get_neighbors": _tool_get_neighbors, + "get_community": _tool_get_community, + "god_nodes": _tool_god_nodes, + "graph_stats": _tool_graph_stats, + "shortest_path": _tool_shortest_path, + } + @server.call_tool() async def call_tool(name: str, arguments: dict) -> list[types.TextContent]: - if name == "query_graph": - question = arguments["question"] - mode = arguments.get("mode", "bfs") - depth = min(int(arguments.get("depth", 3)), 6) - budget = int(arguments.get("token_budget", 2000)) - terms = [t.lower() for t in question.split() if len(t) > 2] - scored = _score_nodes(G, terms) - start_nodes = [nid for _, nid in scored[:3]] - if not start_nodes: - return [types.TextContent(type="text", text="No matching nodes found.")] - if mode == "dfs": - nodes, edges = _dfs(G, start_nodes, depth) - else: - nodes, edges = _bfs(G, start_nodes, depth) - text = f"Traversal: {mode.upper()} depth={depth} | Start: {[G.nodes[n].get('label', n) for n in start_nodes]} | {len(nodes)} nodes found\n\n" - text += _subgraph_to_text(G, nodes, edges, budget) - return [types.TextContent(type="text", text=text)] - - elif name == "get_node": - label = arguments["label"].lower() - matches = [(nid, d) for nid, d in G.nodes(data=True) - if label in d.get("label", "").lower() or label == nid.lower()] - if not matches: - return [types.TextContent(type="text", text=f"No node matching '{label}' found.")] - nid, d = matches[0] - lines = [f"Node: {d.get('label', nid)}", - f" ID: {nid}", - f" Source: {d.get('source_file', '')} {d.get('source_location', '')}", - f" Type: {d.get('file_type', '')}", - f" Community: {d.get('community', '')}", - f" Degree: {G.degree(nid)}"] - return [types.TextContent(type="text", text="\n".join(lines))] - - elif name == "get_neighbors": - label = arguments["label"].lower() - rel_filter = arguments.get("relation_filter", "").lower() - matches = [nid for nid, d in G.nodes(data=True) - if label in d.get("label", "").lower() or label == nid.lower()] - if not matches: - return [types.TextContent(type="text", text=f"No node matching '{label}' found.")] - nid = matches[0] - lines = [f"Neighbors of {G.nodes[nid].get('label', nid)}:"] - for neighbor in G.neighbors(nid): - d = G.edges[nid, neighbor] - rel = d.get("relation", "") - if rel_filter and rel_filter not in rel.lower(): - continue - conf = d.get("confidence", "") - nlabel = G.nodes[neighbor].get("label", neighbor) - lines.append(f" --> {nlabel} [{rel}] [{conf}]") - return [types.TextContent(type="text", text="\n".join(lines))] - - elif name == "get_community": - cid = int(arguments["community_id"]) - nodes = communities.get(cid, []) - if not nodes: - return [types.TextContent(type="text", text=f"Community {cid} not found.")] - lines = [f"Community {cid} ({len(nodes)} nodes):"] - for n in nodes: - d = G.nodes[n] - lines.append(f" {d.get('label', n)} [{d.get('source_file', '')}]") - return [types.TextContent(type="text", text="\n".join(lines))] - - elif name == "god_nodes": - from .analyze import god_nodes as _god_nodes - top_n = int(arguments.get("top_n", 10)) - nodes = _god_nodes(G, top_n=top_n) - lines = ["God nodes (most connected):"] - for i, n in enumerate(nodes, 1): - lines.append(f" {i}. {n['label']} - {n['edges']} edges") - return [types.TextContent(type="text", text="\n".join(lines))] - - elif name == "graph_stats": - confs = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)] - total = len(confs) or 1 - text = ( - f"Nodes: {G.number_of_nodes()}\n" - f"Edges: {G.number_of_edges()}\n" - f"Communities: {len(communities)}\n" - f"EXTRACTED: {round(confs.count('EXTRACTED')/total*100)}%\n" - f"INFERRED: {round(confs.count('INFERRED')/total*100)}%\n" - f"AMBIGUOUS: {round(confs.count('AMBIGUOUS')/total*100)}%\n" - ) - return [types.TextContent(type="text", text=text)] - - elif name == "shortest_path": - src_terms = [t.lower() for t in arguments["source"].split()] - tgt_terms = [t.lower() for t in arguments["target"].split()] - max_hops = int(arguments.get("max_hops", 8)) - src_scored = _score_nodes(G, src_terms) - tgt_scored = _score_nodes(G, tgt_terms) - if not src_scored: - return [types.TextContent(type="text", text=f"No node matching source '{arguments['source']}' found.")] - if not tgt_scored: - return [types.TextContent(type="text", text=f"No node matching target '{arguments['target']}' found.")] - src_nid = src_scored[0][1] - tgt_nid = tgt_scored[0][1] - try: - path_nodes = nx.shortest_path(G, src_nid, tgt_nid) - except (nx.NetworkXNoPath, nx.NodeNotFound): - src_label = G.nodes[src_nid].get("label", src_nid) - tgt_label = G.nodes[tgt_nid].get("label", tgt_nid) - return [types.TextContent(type="text", text=f"No path found between '{src_label}' and '{tgt_label}'.")] - hops = len(path_nodes) - 1 - if hops > max_hops: - return [types.TextContent(type="text", text=f"Path exceeds max_hops={max_hops} ({hops} hops found).")] - segments = [] - for i in range(len(path_nodes) - 1): - u, v = path_nodes[i], path_nodes[i + 1] - u_label = G.nodes[u].get("label", u) - v_label = G.nodes[v].get("label", v) - edata = G.edges[u, v] - rel = edata.get("relation", "") - conf = edata.get("confidence", "") - conf_str = f" [{conf}]" if conf else "" - if i == 0: - segments.append(f"{u_label}") - segments.append(f"--{rel}{conf_str}--> {v_label}") - text = f"Shortest path ({hops} hops):\n " + " ".join(segments) - return [types.TextContent(type="text", text=text)] - - return [types.TextContent(type="text", text=f"Unknown tool: {name}")] + handler = _handlers.get(name) + if not handler: + return [types.TextContent(type="text", text=f"Unknown tool: {name}")] + return [types.TextContent(type="text", text=handler(arguments))] import asyncio