feat: Claude Code skill, Obsidian vault, install, tests

skill.md with full pipeline steps, Obsidian as default output (canvas, tags,
dataview, graph colors), two-command install, 71 tests, .gitignore, deps

graphify/__init__.py

@@ -0,0 +1,7 @@
+"""graphify — extract · build · cluster · analyze · report."""
+from graphify.extract import extract, collect_files
+from graphify.build import build_from_json
+from graphify.cluster import cluster, score_all, cohesion_score
+from graphify.analyze import god_nodes, surprising_connections, suggest_questions
+from graphify.report import generate
+from graphify.export import to_json, to_html, to_svg

graphify/analyze.py

@@ -0,0 +1,417 @@
+"""Graph analysis: god nodes (most connected), surprising connections (cross-community), suggested questions."""
+from __future__ import annotations
+import networkx as nx
+
+
+def _is_file_node(G: nx.Graph, node_id: str) -> bool:
+    """
+    Return True if this node is a file-level hub node (e.g. 'client', 'models')
+    or an AST method stub (e.g. '.auth_flow()', '.__init__()').
+
+    These are synthetic nodes created by the AST extractor and should be excluded
+    from god nodes, surprising connections, and knowledge gap reporting.
+    """
+    label = G.nodes[node_id].get("label", "")
+    if not label:
+        return False
+    # File-level hub: label is a filename with a code extension
+    if label.split(".")[-1] in ("py", "ts", "js", "go", "rs", "java", "rb", "cpp", "c", "h"):
+        return True
+    # Method stub: AST extractor labels methods as '.method_name()'
+    if label.startswith(".") and label.endswith("()"):
+        return True
+    # Module-level function stub: labeled 'function_name()' — only has a contains edge
+    # These are real functions but structurally isolated by definition; not a gap worth flagging
+    if label.endswith("()") and G.degree(node_id) <= 1:
+        return True
+    return False
+
+
+def god_nodes(G: nx.Graph, top_n: int = 10) -> list[dict]:
+    """Return the top_n most-connected real entities — the core abstractions.
+
+    File-level hub nodes are excluded: they accumulate import/contains edges
+    mechanically and don't represent meaningful architectural abstractions.
+    """
+    degree = dict(G.degree())
+    sorted_nodes = sorted(degree.items(), key=lambda x: x[1], reverse=True)
+    result = []
+    for node_id, deg in sorted_nodes:
+        if _is_file_node(G, node_id) or _is_concept_node(G, node_id):
+            continue
+        result.append({
+            "id": node_id,
+            "label": G.nodes[node_id].get("label", node_id),
+            "edges": deg,
+        })
+        if len(result) >= top_n:
+            break
+    return result
+
+
+def surprising_connections(
+    G: nx.Graph,
+    communities: dict[int, list[str]] | None = None,
+    top_n: int = 5,
+) -> list[dict]:
+    """
+    Find connections that are genuinely surprising — not obvious from file structure.
+
+    Strategy:
+    - Multi-file corpora: cross-file edges between real entities (not concept nodes).
+      Sorted AMBIGUOUS → INFERRED → EXTRACTED.
+    - Single-file / single-source corpora: cross-community edges that bridge
+      distant parts of the graph (betweenness centrality on edges).
+      These reveal non-obvious structural couplings.
+
+    Concept nodes (empty source_file, or injected semantic annotations) are excluded
+    from surprising connections because they are intentional, not discovered.
+    """
+    # Identify unique source files (ignore empty/null source_file)
+    source_files = {
+        data.get("source_file", "")
+        for _, data in G.nodes(data=True)
+        if data.get("source_file", "")
+    }
+    is_multi_source = len(source_files) > 1
+
+    if is_multi_source:
+        return _cross_file_surprises(G, communities or {}, top_n)
+    else:
+        return _cross_community_surprises(G, communities or {}, top_n)
+
+
+def _is_concept_node(G: nx.Graph, node_id: str) -> bool:
+    """
+    Return True if this node is a manually-injected semantic concept node
+    rather than a real entity found in source code.
+
+    Signals:
+    - Empty source_file
+    - source_file doesn't look like a real file path (no extension)
+    """
+    data = G.nodes[node_id]
+    source = data.get("source_file", "")
+    if not source:
+        return True
+    # Has no file extension → probably a concept label, not a real file
+    if "." not in source.split("/")[-1]:
+        return True
+    return False
+
+
+def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: int) -> list[dict]:
+    """
+    Cross-file edges between real code/doc entities.
+    Excludes concept nodes, file hub nodes, and plain import edges.
+    Sorted AMBIGUOUS → INFERRED → EXTRACTED.
+    """
+    surprises = []
+    order = {"AMBIGUOUS": 0, "INFERRED": 1, "EXTRACTED": 2}
+
+    for u, v, data in G.edges(data=True):
+        # Skip structural scaffolding — not insights
+        relation = data.get("relation", "")
+        if relation in ("imports", "imports_from", "contains", "method"):
+            continue
+        # Skip if either endpoint is a concept or file-level node
+        if _is_concept_node(G, u) or _is_concept_node(G, v):
+            continue
+        if _is_file_node(G, u) or _is_file_node(G, v):
+            continue
+
+        u_source = G.nodes[u].get("source_file", "")
+        v_source = G.nodes[v].get("source_file", "")
+
+        if u_source and v_source and u_source != v_source:
+            # Respect original edge direction stored in _src/_tgt (if present),
+            # otherwise fall back to u/v which may be in arbitrary order.
+            src_id = data.get("_src", u)
+            tgt_id = data.get("_tgt", v)
+            surprises.append({
+                "source": G.nodes[src_id].get("label", src_id),
+                "target": G.nodes[tgt_id].get("label", tgt_id),
+                "source_files": [
+                    G.nodes[src_id].get("source_file", ""),
+                    G.nodes[tgt_id].get("source_file", ""),
+                ],
+                "confidence": data.get("confidence", "EXTRACTED"),
+                "relation": relation,
+            })
+
+    surprises.sort(key=lambda x: order.get(x["confidence"], 3))
+    if surprises:
+        return surprises[:top_n]
+
+    # Fallback: no semantic cross-file edges found (pure AST corpus).
+    # Surface cross-community bridge edges as structural surprises instead.
+    return _cross_community_surprises(G, communities, top_n)
+
+
+def _cross_community_surprises(
+    G: nx.Graph,
+    communities: dict[int, list[str]],
+    top_n: int,
+) -> list[dict]:
+    """
+    For single-source corpora: find edges that bridge different communities.
+    These are surprising because Leiden grouped everything else tightly —
+    these edges cut across the natural structure.
+
+    Falls back to high-betweenness edges if no community info is provided.
+    """
+    if not communities:
+        # No community info — use edge betweenness centrality
+        if G.number_of_edges() == 0:
+            return []
+        betweenness = nx.edge_betweenness_centrality(G)
+        top_edges = sorted(betweenness.items(), key=lambda x: x[1], reverse=True)[:top_n]
+        result = []
+        for (u, v), score in top_edges:
+            data = G.edges[u, v]
+            result.append({
+                "source": G.nodes[u].get("label", u),
+                "target": G.nodes[v].get("label", v),
+                "source_files": [
+                    G.nodes[u].get("source_file", ""),
+                    G.nodes[v].get("source_file", ""),
+                ],
+                "confidence": data.get("confidence", "EXTRACTED"),
+                "relation": data.get("relation", ""),
+                "note": f"Bridges graph structure (betweenness={score:.3f})",
+            })
+        return result
+
+    # Build node → community map
+    node_community = {n: cid for cid, nodes in communities.items() for n in nodes}
+
+    surprises = []
+    for u, v, data in G.edges(data=True):
+        cid_u = node_community.get(u)
+        cid_v = node_community.get(v)
+        if cid_u is None or cid_v is None or cid_u == cid_v:
+            continue
+        # Skip file hub nodes and plain structural edges
+        if _is_file_node(G, u) or _is_file_node(G, v):
+            continue
+        relation = data.get("relation", "")
+        if relation in ("imports", "imports_from", "contains", "method"):
+            continue
+        # This edge crosses community boundaries — interesting
+        confidence = data.get("confidence", "EXTRACTED")
+        src_id = data.get("_src", u)
+        tgt_id = data.get("_tgt", v)
+        surprises.append({
+            "source": G.nodes[src_id].get("label", src_id),
+            "target": G.nodes[tgt_id].get("label", tgt_id),
+            "source_files": [
+                G.nodes[src_id].get("source_file", ""),
+                G.nodes[tgt_id].get("source_file", ""),
+            ],
+            "confidence": confidence,
+            "relation": relation,
+            "note": f"Bridges community {cid_u} → community {cid_v}",
+            "_pair": tuple(sorted([cid_u, cid_v])),
+        })
+
+    # Sort: AMBIGUOUS first, then INFERRED, then EXTRACTED
+    order = {"AMBIGUOUS": 0, "INFERRED": 1, "EXTRACTED": 2}
+    surprises.sort(key=lambda x: order.get(x["confidence"], 3))
+
+    # Deduplicate by community pair — one representative edge per (A→B) boundary.
+    # Without this, a single high-betweenness god node dominates all results.
+    seen_pairs: set[tuple] = set()
+    deduped = []
+    for s in surprises:
+        pair = s.pop("_pair")
+        if pair not in seen_pairs:
+            seen_pairs.add(pair)
+            deduped.append(s)
+    return deduped[:top_n]
+
+
+def suggest_questions(
+    G: nx.Graph,
+    communities: dict[int, list[str]],
+    community_labels: dict[int, str],
+    top_n: int = 7,
+) -> list[dict]:
+    """
+    Generate questions the graph is uniquely positioned to answer.
+    Based on: AMBIGUOUS edges, bridge nodes, underexplored god nodes, isolated nodes.
+    Each question has a 'type', 'question', and 'why' field.
+    """
+    questions = []
+    node_community = {n: cid for cid, nodes in communities.items() for n in nodes}
+
+    # 1. AMBIGUOUS edges → unresolved relationship questions
+    for u, v, data in G.edges(data=True):
+        if data.get("confidence") == "AMBIGUOUS":
+            ul = G.nodes[u].get("label", u)
+            vl = G.nodes[v].get("label", v)
+            relation = data.get("relation", "related to")
+            questions.append({
+                "type": "ambiguous_edge",
+                "question": f"What is the exact relationship between `{ul}` and `{vl}`?",
+                "why": f"Edge tagged AMBIGUOUS (relation: {relation}) — confidence is low.",
+            })
+
+    # 2. Bridge nodes (high betweenness) → cross-cutting concern questions
+    if G.number_of_edges() > 0:
+        betweenness = nx.betweenness_centrality(G)
+        # Top bridge nodes that are NOT file-level hubs
+        bridges = sorted(
+            [(n, s) for n, s in betweenness.items()
+             if not _is_file_node(G, n) and not _is_concept_node(G, n) and s > 0],
+            key=lambda x: x[1],
+            reverse=True,
+        )[:3]
+        for node_id, score in bridges:
+            label = G.nodes[node_id].get("label", node_id)
+            cid = node_community.get(node_id)
+            comm_label = community_labels.get(cid, f"Community {cid}") if cid is not None else "unknown"
+            neighbors = list(G.neighbors(node_id))
+            neighbor_comms = {node_community.get(n) for n in neighbors if node_community.get(n) != cid}
+            if neighbor_comms:
+                other_labels = [community_labels.get(c, f"Community {c}") for c in neighbor_comms]
+                questions.append({
+                    "type": "bridge_node",
+                    "question": f"Why does `{label}` connect `{comm_label}` to {', '.join(f'`{l}`' for l in other_labels)}?",
+                    "why": f"High betweenness centrality ({score:.3f}) — this node is a cross-community bridge.",
+                })
+
+    # 3. God nodes with many INFERRED edges → verification questions
+    degree = dict(G.degree())
+    top_nodes = sorted(
+        [(n, d) for n, d in degree.items() if not _is_file_node(G, n)],
+        key=lambda x: x[1],
+        reverse=True,
+    )[:5]
+    for node_id, _ in top_nodes:
+        inferred = [
+            (u, v, d) for u, v, d in G.edges(node_id, data=True)
+            if d.get("confidence") == "INFERRED"
+        ]
+        if len(inferred) >= 2:
+            label = G.nodes[node_id].get("label", node_id)
+            # Use _src/_tgt to get the correct direction; fall back to v (the other node)
+            others = []
+            for u, v, d in inferred[:2]:
+                src_id = d.get("_src", u)
+                tgt_id = d.get("_tgt", v)
+                other_id = tgt_id if src_id == node_id else src_id
+                others.append(G.nodes[other_id].get("label", other_id))
+            questions.append({
+                "type": "verify_inferred",
+                "question": f"Are the {len(inferred)} inferred relationships involving `{label}` (e.g. with `{others[0]}` and `{others[1]}`) actually correct?",
+                "why": f"`{label}` has {len(inferred)} INFERRED edges — model-reasoned connections that need verification.",
+            })
+
+    # 4. Isolated or weakly-connected nodes → exploration questions
+    isolated = [
+        n for n in G.nodes()
+        if G.degree(n) <= 1 and not _is_file_node(G, n) and not _is_concept_node(G, n)
+    ]
+    if isolated:
+        labels = [G.nodes[n].get("label", n) for n in isolated[:3]]
+        questions.append({
+            "type": "isolated_nodes",
+            "question": f"What connects {', '.join(f'`{l}`' for l in labels)} to the rest of the system?",
+            "why": f"{len(isolated)} weakly-connected nodes found — possible documentation gaps or missing edges.",
+        })
+
+    # 5. Low-cohesion communities → structural questions
+    from .cluster import cohesion_score
+    for cid, nodes in communities.items():
+        score = cohesion_score(G, nodes)
+        if score < 0.15 and len(nodes) >= 5:
+            label = community_labels.get(cid, f"Community {cid}")
+            questions.append({
+                "type": "low_cohesion",
+                "question": f"Should `{label}` be split into smaller, more focused modules?",
+                "why": f"Cohesion score {score} — nodes in this community are weakly interconnected.",
+            })
+
+    return questions[:top_n]
+
+
+def graph_diff(G_old: nx.Graph, G_new: nx.Graph) -> dict:
+    """Compare two graph snapshots and return what changed.
+
+    Returns:
+        {
+          "new_nodes": [{"id": ..., "label": ...}],
+          "removed_nodes": [{"id": ..., "label": ...}],
+          "new_edges": [{"source": ..., "target": ..., "relation": ..., "confidence": ...}],
+          "removed_edges": [...],
+          "summary": "3 new nodes, 5 new edges, 1 node removed"
+        }
+    """
+    old_nodes = set(G_old.nodes())
+    new_nodes = set(G_new.nodes())
+
+    added_node_ids = new_nodes - old_nodes
+    removed_node_ids = old_nodes - new_nodes
+
+    new_nodes_list = [
+        {"id": n, "label": G_new.nodes[n].get("label", n)}
+        for n in added_node_ids
+    ]
+    removed_nodes_list = [
+        {"id": n, "label": G_old.nodes[n].get("label", n)}
+        for n in removed_node_ids
+    ]
+
+    def edge_key(G: nx.Graph, u: str, v: str, data: dict) -> tuple:
+        return (u, v, data.get("relation", ""))
+
+    old_edge_keys = {
+        edge_key(G_old, u, v, d)
+        for u, v, d in G_old.edges(data=True)
+    }
+    new_edge_keys = {
+        edge_key(G_new, u, v, d)
+        for u, v, d in G_new.edges(data=True)
+    }
+
+    added_edge_keys = new_edge_keys - old_edge_keys
+    removed_edge_keys = old_edge_keys - new_edge_keys
+
+    new_edges_list = []
+    for u, v, d in G_new.edges(data=True):
+        if edge_key(G_new, u, v, d) in added_edge_keys:
+            new_edges_list.append({
+                "source": u,
+                "target": v,
+                "relation": d.get("relation", ""),
+                "confidence": d.get("confidence", ""),
+            })
+
+    removed_edges_list = []
+    for u, v, d in G_old.edges(data=True):
+        if edge_key(G_old, u, v, d) in removed_edge_keys:
+            removed_edges_list.append({
+                "source": u,
+                "target": v,
+                "relation": d.get("relation", ""),
+                "confidence": d.get("confidence", ""),
+            })
+
+    parts = []
+    if new_nodes_list:
+        parts.append(f"{len(new_nodes_list)} new node{'s' if len(new_nodes_list) != 1 else ''}")
+    if new_edges_list:
+        parts.append(f"{len(new_edges_list)} new edge{'s' if len(new_edges_list) != 1 else ''}")
+    if removed_nodes_list:
+        parts.append(f"{len(removed_nodes_list)} node{'s' if len(removed_nodes_list) != 1 else ''} removed")
+    if removed_edges_list:
+        parts.append(f"{len(removed_edges_list)} edge{'s' if len(removed_edges_list) != 1 else ''} removed")
+    summary = ", ".join(parts) if parts else "no changes"
+
+    return {
+        "new_nodes": new_nodes_list,
+        "removed_nodes": removed_nodes_list,
+        "new_edges": new_edges_list,
+        "removed_edges": removed_edges_list,
+        "summary": summary,
+    }

graphify/build.py

@@ -0,0 +1,26 @@
+# assemble node+edge dicts into a NetworkX graph, preserving edge direction
+from __future__ import annotations
+import networkx as nx
+
+
+def build_from_json(extraction: dict) -> nx.Graph:
+    G = nx.Graph()
+    for node in extraction.get("nodes", []):
+        G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"})
+    for edge in extraction.get("edges", []):
+        attrs = {k: v for k, v in edge.items() if k not in ("source", "target")}
+        # Preserve original edge direction — undirected graphs lose it otherwise,
+        # causing display functions to show edges backwards.
+        attrs["_src"] = edge["source"]
+        attrs["_tgt"] = edge["target"]
+        G.add_edge(edge["source"], edge["target"], **attrs)
+    return G
+
+
+def build(extractions: list[dict]) -> nx.Graph:
+    """Merge multiple extraction results into one graph."""
+    G = nx.Graph()
+    for ext in extractions:
+        sub = build_from_json(ext)
+        G.update(sub)
+    return G

graphify/cache.py

@@ -0,0 +1,63 @@
+# per-file extraction cache — skip unchanged files on re-run
+from __future__ import annotations
+
+import hashlib
+import json
+import shutil
+from pathlib import Path
+
+
+def file_hash(path: Path) -> str:
+    """SHA256 of file contents, hex digest."""
+    return hashlib.sha256(Path(path).read_bytes()).hexdigest()
+
+
+def cache_dir(root: Path = Path(".")) -> Path:
+    """Returns .graphify/cache/ — creates it if needed."""
+    d = Path(root) / ".graphify" / "cache"
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def load_cached(path: Path, root: Path = Path(".")) -> dict | None:
+    """Return cached extraction for this file if hash matches, else None.
+
+    Cache key: SHA256 of file contents.
+    Cache value: stored as .graphify/cache/{hash}.json
+    Returns None if no cache entry or file has changed.
+    """
+    try:
+        h = file_hash(path)
+    except OSError:
+        return None
+    entry = cache_dir(root) / f"{h}.json"
+    if not entry.exists():
+        return None
+    try:
+        return json.loads(entry.read_text())
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+def save_cached(path: Path, result: dict, root: Path = Path(".")) -> None:
+    """Save extraction result for this file.
+
+    Stores as .graphify/cache/{hash}.json where hash = SHA256 of current file contents.
+    result should be a dict with 'nodes' and 'edges' lists.
+    """
+    h = file_hash(path)
+    entry = cache_dir(root) / f"{h}.json"
+    entry.write_text(json.dumps(result))
+
+
+def cached_files(root: Path = Path(".")) -> set[str]:
+    """Return set of file paths that have a valid cache entry (hash still matches)."""
+    d = cache_dir(root)
+    return {p.stem for p in d.glob("*.json")}
+
+
+def clear_cache(root: Path = Path(".")) -> None:
+    """Delete all .graphify/cache/*.json files."""
+    d = cache_dir(root)
+    for f in d.glob("*.json"):
+        f.unlink()

graphify/cluster.py

@@ -0,0 +1,90 @@
+"""Leiden community detection on NetworkX graphs. Splits oversized communities. Returns cohesion scores."""
+from __future__ import annotations
+import networkx as nx
+from graspologic.partition import leiden
+
+
+def build_graph(nodes: list[dict], edges: list[dict]) -> nx.Graph:
+    """Build a NetworkX graph from graphify node/edge dicts.
+
+    Preserves original edge direction as _src/_tgt attributes so that
+    display functions can show relationships in the correct direction,
+    even though the graph is undirected for structural analysis.
+    """
+    G = nx.Graph()
+    for n in nodes:
+        G.add_node(n["id"], **{k: v for k, v in n.items() if k != "id"})
+    for e in edges:
+        attrs = {k: v for k, v in e.items() if k not in ("source", "target")}
+        attrs["_src"] = e["source"]
+        attrs["_tgt"] = e["target"]
+        G.add_edge(e["source"], e["target"], **attrs)
+    return G
+
+_MAX_COMMUNITY_FRACTION = 0.25   # communities larger than 25% of graph get split
+_MIN_SPLIT_SIZE = 10             # only split if community has at least this many nodes
+
+
+def cluster(G: nx.Graph) -> dict[int, list[str]]:
+    """Run Leiden community detection. Returns {community_id: [node_ids]}.
+
+    Community IDs are stable across runs: 0 = largest community after splitting.
+    Oversized communities (> 25% of graph nodes, min 10) are split by running
+    a second Leiden pass on the subgraph.
+    """
+    if G.number_of_nodes() == 0:
+        return {}
+    if G.number_of_edges() == 0:
+        return {i: [n] for i, n in enumerate(sorted(G.nodes))}
+
+    partition: dict[str, int] = leiden(G)
+    raw: dict[int, list[str]] = {}
+    for node, cid in partition.items():
+        raw.setdefault(cid, []).append(node)
+
+    # Split oversized communities
+    max_size = max(_MIN_SPLIT_SIZE, int(G.number_of_nodes() * _MAX_COMMUNITY_FRACTION))
+    final_communities: list[list[str]] = []
+    for nodes in raw.values():
+        if len(nodes) > max_size:
+            final_communities.extend(_split_community(G, nodes))
+        else:
+            final_communities.append(nodes)
+
+    # Re-index by size descending for deterministic ordering
+    final_communities.sort(key=len, reverse=True)
+    return {i: sorted(nodes) for i, nodes in enumerate(final_communities)}
+
+
+def _split_community(G: nx.Graph, nodes: list[str]) -> list[list[str]]:
+    """Run a second Leiden pass on a community subgraph to split it further."""
+    subgraph = G.subgraph(nodes)
+    if subgraph.number_of_edges() == 0:
+        # No edges — split into individual nodes
+        return [[n] for n in sorted(nodes)]
+    try:
+        sub_partition: dict[str, int] = leiden(subgraph)
+        sub_communities: dict[int, list[str]] = {}
+        for node, cid in sub_partition.items():
+            sub_communities.setdefault(cid, []).append(node)
+        if len(sub_communities) <= 1:
+            # Leiden couldn't split it — return as-is
+            return [sorted(nodes)]
+        return [sorted(v) for v in sub_communities.values()]
+    except Exception:
+        return [sorted(nodes)]
+
+
+def cohesion_score(G: nx.Graph, community_nodes: list[str]) -> float:
+    """Ratio of actual intra-community edges to maximum possible."""
+    n = len(community_nodes)
+    if n <= 1:
+        return 1.0
+    subgraph = G.subgraph(community_nodes)
+    actual = subgraph.number_of_edges()
+    possible = n * (n - 1) / 2
+    return round(actual / possible, 2) if possible > 0 else 0.0
+
+
+def score_all(G: nx.Graph, communities: dict[int, list[str]]) -> dict[int, float]:
+    return {cid: cohesion_score(G, nodes) for cid, nodes in communities.items()}

graphify/detect.py

@@ -0,0 +1,247 @@
+# file discovery, type classification, and corpus health checks
+from __future__ import annotations
+import json
+import re
+from enum import Enum
+from pathlib import Path
+
+
+class FileType(str, Enum):
+    CODE = "code"
+    DOCUMENT = "document"
+    PAPER = "paper"
+    IMAGE = "image"
+
+
+_MANIFEST_PATH = ".graphify/manifest.json"
+
+CODE_EXTENSIONS = {'.py', '.ts', '.js', '.go', '.rs', '.java', '.cpp', '.c', '.rb', '.swift', '.kt'}
+DOC_EXTENSIONS = {'.md', '.txt', '.rst'}
+PAPER_EXTENSIONS = {'.pdf'}
+IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}
+
+CORPUS_WARN_THRESHOLD = 50_000    # words — below this, warn "you may not need a graph"
+CORPUS_UPPER_THRESHOLD = 500_000  # words — above this, warn about token cost
+FILE_COUNT_UPPER = 200             # files — above this, warn about token cost
+
+# Files that may contain secrets — skip silently
+_SENSITIVE_PATTERNS = [
+    re.compile(r'(^|[\\/])\.(env|envrc)(\.|$)', re.IGNORECASE),
+    re.compile(r'\.(pem|key|p12|pfx|cert|crt|der|p8)$', re.IGNORECASE),
+    re.compile(r'(credential|secret|passwd|password|token|private_key)', re.IGNORECASE),
+    re.compile(r'(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.pub)?$'),
+    re.compile(r'(\.netrc|\.pgpass|\.htpasswd)$', re.IGNORECASE),
+    re.compile(r'(aws_credentials|gcloud_credentials|service.account)', re.IGNORECASE),
+]
+
+# Signals that a .md/.txt file is actually a converted academic paper
+_PAPER_SIGNALS = [
+    re.compile(r'\barxiv\b', re.IGNORECASE),
+    re.compile(r'\bdoi\s*:', re.IGNORECASE),
+    re.compile(r'\babstract\b', re.IGNORECASE),
+    re.compile(r'\bproceedings\b', re.IGNORECASE),
+    re.compile(r'\bjournal\b', re.IGNORECASE),
+    re.compile(r'\bpreprint\b', re.IGNORECASE),
+    re.compile(r'\\cite\{'),          # LaTeX citation
+    re.compile(r'\[\d+\]'),           # Numbered citation [1], [23] (inline)
+    re.compile(r'\[\n\d+\n\]'),       # Numbered citation spread across lines (markdown conversion)
+    re.compile(r'eq\.\s*\d+|equation\s+\d+', re.IGNORECASE),
+    re.compile(r'\d{4}\.\d{4,5}'),   # arXiv ID like 1706.03762
+    re.compile(r'\bwe propose\b', re.IGNORECASE),   # common academic phrasing
+    re.compile(r'\bliterature\b', re.IGNORECASE),   # "from the literature"
+]
+_PAPER_SIGNAL_THRESHOLD = 3  # need at least this many signals to call it a paper
+
+
+def _is_sensitive(path: Path) -> bool:
+    """Return True if this file likely contains secrets and should be skipped."""
+    name = path.name
+    full = str(path)
+    return any(p.search(name) or p.search(full) for p in _SENSITIVE_PATTERNS)
+
+
+def _looks_like_paper(path: Path) -> bool:
+    """Heuristic: does this text file read like an academic paper?"""
+    try:
+        # Only scan first 3000 chars for speed
+        text = path.read_text(errors="ignore")[:3000]
+        hits = sum(1 for pattern in _PAPER_SIGNALS if pattern.search(text))
+        return hits >= _PAPER_SIGNAL_THRESHOLD
+    except Exception:
+        return False
+
+
+def classify_file(path: Path) -> FileType | None:
+    ext = path.suffix.lower()
+    if ext in CODE_EXTENSIONS:
+        return FileType.CODE
+    if ext in PAPER_EXTENSIONS:
+        return FileType.PAPER
+    if ext in IMAGE_EXTENSIONS:
+        return FileType.IMAGE
+    if ext in DOC_EXTENSIONS:
+        # Check if it's a converted paper
+        if _looks_like_paper(path):
+            return FileType.PAPER
+        return FileType.DOCUMENT
+    return None
+
+
+def extract_pdf_text(path: Path) -> str:
+    """Extract plain text from a PDF file using pypdf."""
+    try:
+        from pypdf import PdfReader
+        reader = PdfReader(str(path))
+        pages = []
+        for page in reader.pages:
+            text = page.extract_text()
+            if text:
+                pages.append(text)
+        return "\n".join(pages)
+    except Exception:
+        return ""
+
+
+def count_words(path: Path) -> int:
+    try:
+        if path.suffix.lower() == ".pdf":
+            return len(extract_pdf_text(path).split())
+        return len(path.read_text(errors="ignore").split())
+    except Exception:
+        return 0
+
+
+# Directory names to always skip — venvs, caches, build artifacts, deps
+_SKIP_DIRS = {
+    "venv", ".venv", "env", ".env",
+    "node_modules", "__pycache__", ".git",
+    "dist", "build", "target", "out",
+    "site-packages", "lib64",
+    ".pytest_cache", ".mypy_cache", ".ruff_cache",
+    ".tox", ".eggs", "*.egg-info",
+}
+
+def _is_noise_dir(part: str) -> bool:
+    """Return True if this directory name looks like a venv, cache, or dep dir."""
+    if part in _SKIP_DIRS:
+        return True
+    # Catch *_venv, *_repo/site-packages patterns
+    if part.endswith("_venv") or part.endswith("_env"):
+        return True
+    if part.endswith(".egg-info"):
+        return True
+    return False
+
+
+def detect(root: Path) -> dict:
+    files: dict[FileType, list[str]] = {
+        FileType.CODE: [],
+        FileType.DOCUMENT: [],
+        FileType.PAPER: [],
+        FileType.IMAGE: [],
+    }
+    total_words = 0
+
+    skipped_sensitive: list[str] = []
+
+    for p in sorted(root.rglob("*")):
+        if not p.is_file():
+            continue
+        parts = p.relative_to(root).parts
+        # Skip hidden dirs and known noise dirs
+        if any(part.startswith(".") or _is_noise_dir(part) for part in parts):
+            continue
+        if _is_sensitive(p):
+            skipped_sensitive.append(str(p))
+            continue
+        ftype = classify_file(p)
+        if ftype:
+            files[ftype].append(str(p))
+            total_words += count_words(p)
+
+    total_files = sum(len(v) for v in files.values())
+    needs_graph = total_words >= CORPUS_WARN_THRESHOLD
+
+    # Determine warning — lower bound, upper bound, or sensitive files skipped
+    warning: str | None = None
+    if not needs_graph:
+        warning = (
+            f"Corpus is ~{total_words:,} words — fits in a single context window. "
+            f"You may not need a graph."
+        )
+    elif total_words >= CORPUS_UPPER_THRESHOLD or total_files >= FILE_COUNT_UPPER:
+        warning = (
+            f"Large corpus: {total_files} files · ~{total_words:,} words. "
+            f"Semantic extraction will be expensive (many Claude tokens). "
+            f"Consider running on a subfolder, or use --no-semantic to run AST-only."
+        )
+
+    return {
+        "files": {k.value: v for k, v in files.items()},
+        "total_files": total_files,
+        "total_words": total_words,
+        "needs_graph": needs_graph,
+        "warning": warning,
+        "skipped_sensitive": skipped_sensitive,
+    }
+
+
+def load_manifest(manifest_path: str = _MANIFEST_PATH) -> dict[str, float]:
+    """Load the file modification time manifest from a previous run."""
+    try:
+        return json.loads(Path(manifest_path).read_text())
+    except Exception:
+        return {}
+
+
+def save_manifest(files: dict[str, list[str]], manifest_path: str = _MANIFEST_PATH) -> None:
+    """Save current file mtimes so the next --update run can diff against them."""
+    manifest: dict[str, float] = {}
+    for file_list in files.values():
+        for f in file_list:
+            try:
+                manifest[f] = Path(f).stat().st_mtime
+            except Exception:
+                pass
+    Path(manifest_path).parent.mkdir(parents=True, exist_ok=True)
+    Path(manifest_path).write_text(json.dumps(manifest, indent=2))
+
+
+def detect_incremental(root: Path, manifest_path: str = _MANIFEST_PATH) -> dict:
+    """Like detect(), but returns only new or modified files since the last run.
+
+    Compares current file mtimes against the stored manifest.
+    Use for --update mode: re-extract only what changed, merge into existing graph.
+    """
+    full = detect(root)
+    manifest = load_manifest(manifest_path)
+
+    if not manifest:
+        # No previous run — treat everything as new
+        full["incremental"] = True
+        full["new_files"] = full["files"]
+        full["unchanged_files"] = {k: [] for k in full["files"]}
+        full["new_total"] = full["total_files"]
+        return full
+
+    new_files: dict[str, list[str]] = {k: [] for k in full["files"]}
+    unchanged_files: dict[str, list[str]] = {k: [] for k in full["files"]}
+
+    for ftype, file_list in full["files"].items():
+        for f in file_list:
+            stored_mtime = manifest.get(f)
+            try:
+                current_mtime = Path(f).stat().st_mtime
+            except Exception:
+                current_mtime = 0
+            if stored_mtime is None or current_mtime > stored_mtime:
+                new_files[ftype].append(f)
+            else:
+                unchanged_files[ftype].append(f)
+
+    new_total = sum(len(v) for v in new_files.values())
+    full["incremental"] = True
+    full["new_files"] = new_files
+    full["unchanged_files"] = unchanged_files
+    full["new_total"] = new_total
+    return full

graphify/export.py

@@ -0,0 +1,438 @@
+# write graph to HTML, JSON, SVG, Obsidian vault, and Neo4j Cypher
+from __future__ import annotations
+import json
+import re
+from pathlib import Path
+import networkx as nx
+from networkx.readwrite import json_graph
+
+COMMUNITY_COLORS = [
+    "#4E79A7", "#F28E2B", "#E15759", "#76B7B2", "#59A14F",
+    "#EDC948", "#B07AA1", "#FF9DA7", "#9C755F", "#BAB0AC",
+]
+
+MAX_NODES_FOR_VIZ = 5_000
+
+
+def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str) -> None:
+    node_community = {n: cid for cid, nodes in communities.items() for n in nodes}
+    data = json_graph.node_link_data(G, edges="links")
+    for node in data["nodes"]:
+        node["community"] = node_community.get(node["id"])
+    with open(output_path, "w") as f:
+        json.dump(data, f, indent=2)
+
+
+def to_cypher(G: nx.Graph, output_path: str) -> None:
+    lines = ["// Neo4j Cypher import — generated by /graphify", ""]
+    for node_id, data in G.nodes(data=True):
+        label = data.get("label", node_id).replace("'", "\\'")
+        ftype = data.get("file_type", "unknown").capitalize()
+        lines.append(f"MERGE (n:{ftype} {{id: '{node_id}', label: '{label}'}});")
+    lines.append("")
+    for u, v, data in G.edges(data=True):
+        rel = data.get("relation", "RELATES_TO").upper().replace(" ", "_").replace("-", "_")
+        conf = data.get("confidence", "EXTRACTED")
+        lines.append(
+            f"MATCH (a {{id: '{u}'}}), (b {{id: '{v}'}}) "
+            f"MERGE (a)-[:{rel} {{confidence: '{conf}'}}]->(b);"
+        )
+    with open(output_path, "w") as f:
+        f.write("\n".join(lines))
+
+
+def to_html(
+    G: nx.Graph,
+    communities: dict[int, list[str]],
+    output_path: str,
+    community_labels: dict[int, str] | None = None,
+) -> None:
+    """Generate an interactive pyvis HTML visualization of the graph.
+
+    Merged from visualizer.py. Raises ValueError if graph exceeds MAX_NODES_FOR_VIZ.
+    """
+    from pyvis.network import Network
+
+    if G.number_of_nodes() > MAX_NODES_FOR_VIZ:
+        raise ValueError(
+            f"Graph has {G.number_of_nodes()} nodes — too large for pyvis. "
+            f"Use --no-viz or reduce input size."
+        )
+
+    node_community = {n: cid for cid, nodes in communities.items() for n in nodes}
+
+    net = Network(height="800px", width="100%", bgcolor="#1a1a2e", font_color="white")
+    net.barnes_hut()
+
+    for node_id, data in G.nodes(data=True):
+        cid = node_community.get(node_id, 0)
+        color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)]
+        net.add_node(
+            node_id,
+            label=data.get("label", node_id),
+            color=color,
+            title=(
+                f"Source: {data.get('source_file', 'unknown')}\n"
+                f"Type: {data.get('file_type', 'unknown')}\n"
+                f"Community: {community_labels.get(cid, str(cid)) if community_labels else cid}"
+            ),
+        )
+
+    for u, v, data in G.edges(data=True):
+        confidence = data.get("confidence", "EXTRACTED")
+        width = {"EXTRACTED": 2, "INFERRED": 1, "AMBIGUOUS": 1}.get(confidence, 1)
+        net.add_edge(
+            u, v,
+            title=f"{data.get('relation', '')} [{confidence}]",
+            width=width,
+            dashes=(confidence != "EXTRACTED"),
+        )
+
+    net.save_graph(output_path)
+
+    # Inject community legend into saved HTML
+    if community_labels:
+        legend_items = ""
+        for cid in sorted(community_labels.keys()):
+            color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)]
+            label = community_labels[cid]
+            n_nodes = len(communities.get(cid, []))
+            legend_items += (
+                f'<div style="margin:4px 0">'
+                f'<span style="color:{color};font-size:18px">■</span> '
+                f'<span style="font-size:13px">{label} ({n_nodes})</span>'
+                f'</div>'
+            )
+        legend_html = (
+            '<div style="position:fixed;top:10px;right:10px;background:#2a2a4e;'
+            'padding:12px 16px;border-radius:8px;font-family:sans-serif;color:white;'
+            'z-index:9999;min-width:180px;">'
+            '<b style="font-size:14px">Communities</b><br>'
+            + legend_items +
+            '</div>'
+        )
+        content = Path(output_path).read_text()
+        content = content.replace("</body>", legend_html + "\n</body>")
+        Path(output_path).write_text(content)
+
+
+# Keep backward-compatible alias — skill.md calls generate_html
+generate_html = to_html
+
+
+def to_obsidian(
+    G: nx.Graph,
+    communities: dict[int, list[str]],
+    output_dir: str,
+    community_labels: dict[int, str] | None = None,
+    cohesion: dict[int, float] | None = None,
+) -> int:
+    """Export graph as an Obsidian vault — one .md file per node with [[wikilinks]],
+    plus one _COMMUNITY_name.md overview note per community (sorted to top by underscore prefix).
+
+    Open the output directory as a vault in Obsidian to get an interactive
+    graph view with community colors and full-text search over node metadata.
+
+    Returns the number of node notes + community notes written.
+    """
+    out = Path(output_dir)
+    out.mkdir(parents=True, exist_ok=True)
+
+    node_community = {n: cid for cid, nodes in communities.items() for n in nodes}
+
+    # Map node_id → safe filename so wikilinks stay consistent.
+    # Deduplicate: if two nodes produce the same filename, append a numeric suffix.
+    def safe_name(label: str) -> str:
+        return re.sub(r'[\\/*?:"<>|#^[\]]', "", label).strip() or "unnamed"
+
+    node_filename: dict[str, str] = {}
+    seen_names: dict[str, int] = {}
+    for node_id, data in G.nodes(data=True):
+        base = safe_name(data.get("label", node_id))
+        if base in seen_names:
+            seen_names[base] += 1
+            node_filename[node_id] = f"{base}_{seen_names[base]}"
+        else:
+            seen_names[base] = 0
+            node_filename[node_id] = base
+
+    # Write one .md file per node
+    for node_id, data in G.nodes(data=True):
+        label = data.get("label", node_id)
+        cid = node_community.get(node_id)
+        community_name = (
+            community_labels.get(cid, f"Community {cid}")
+            if community_labels and cid is not None
+            else f"Community {cid}"
+        )
+
+        lines: list[str] = []
+
+        # YAML frontmatter — readable in Obsidian's properties panel
+        lines += [
+            "---",
+            f'source_file: "{data.get("source_file", "")}"',
+            f'type: "{data.get("file_type", "")}"',
+            f'community: "{community_name}"',
+        ]
+        if data.get("source_location"):
+            lines.append(f'location: "{data["source_location"]}"')
+        lines += ["---", "", f"# {label}", ""]
+
+        # Outgoing edges as wikilinks
+        neighbors = list(G.neighbors(node_id))
+        if neighbors:
+            lines.append("## Connections")
+            for neighbor in sorted(neighbors, key=lambda n: G.nodes[n].get("label", n)):
+                edge_data = G.edges[node_id, neighbor]
+                neighbor_label = node_filename[neighbor]
+                relation = edge_data.get("relation", "")
+                confidence = edge_data.get("confidence", "EXTRACTED")
+                lines.append(f"- [[{neighbor_label}]] — `{relation}` [{confidence}]")
+
+        fname = node_filename[node_id] + ".md"
+        (out / fname).write_text("\n".join(lines), encoding="utf-8")
+
+    # Write one _COMMUNITY_name.md overview note per community
+    # Build inter-community edge counts for "Connections to other communities"
+    inter_community_edges: dict[int, dict[int, int]] = {}
+    for cid in communities:
+        inter_community_edges[cid] = {}
+    for u, v in G.edges():
+        cu = node_community.get(u)
+        cv = node_community.get(v)
+        if cu is not None and cv is not None and cu != cv:
+            inter_community_edges.setdefault(cu, {})
+            inter_community_edges.setdefault(cv, {})
+            inter_community_edges[cu][cv] = inter_community_edges[cu].get(cv, 0) + 1
+            inter_community_edges[cv][cu] = inter_community_edges[cv].get(cu, 0) + 1
+
+    # Precompute per-node community reach (number of distinct communities a node connects to)
+    def _community_reach(node_id: str) -> int:
+        neighbor_cids = {
+            node_community[nb]
+            for nb in G.neighbors(node_id)
+            if nb in node_community and node_community[nb] != node_community.get(node_id)
+        }
+        return len(neighbor_cids)
+
+    community_notes_written = 0
+    for cid, members in communities.items():
+        community_name = (
+            community_labels.get(cid, f"Community {cid}")
+            if community_labels and cid is not None
+            else f"Community {cid}"
+        )
+        n_members = len(members)
+        coh_value = cohesion.get(cid) if cohesion else None
+
+        lines: list[str] = []
+
+        # YAML frontmatter
+        lines.append("---")
+        lines.append("type: community")
+        if coh_value is not None:
+            lines.append(f"cohesion: {coh_value:.2f}")
+        lines.append(f"members: {n_members}")
+        lines.append("---")
+        lines.append("")
+        lines.append(f"# {community_name}")
+        lines.append("")
+
+        # Cohesion + member count summary
+        if coh_value is not None:
+            cohesion_desc = (
+                "tightly connected" if coh_value >= 0.7
+                else "moderately connected" if coh_value >= 0.4
+                else "loosely connected"
+            )
+            lines.append(f"**Cohesion:** {coh_value:.2f} — {cohesion_desc}")
+        lines.append(f"**Members:** {n_members} nodes")
+        lines.append("")
+
+        # Members section
+        lines.append("## Members")
+        for node_id in sorted(members, key=lambda n: G.nodes[n].get("label", n)):
+            data = G.nodes[node_id]
+            node_label = node_filename[node_id]
+            ftype = data.get("file_type", "")
+            source = data.get("source_file", "")
+            entry = f"- [[{node_label}]]"
+            if ftype:
+                entry += f" — {ftype}"
+            if source:
+                entry += f" — {source}"
+            lines.append(entry)
+        lines.append("")
+
+        # Connections to other communities
+        cross = inter_community_edges.get(cid, {})
+        if cross:
+            lines.append("## Connections to other communities")
+            for other_cid, edge_count in sorted(cross.items(), key=lambda x: -x[1]):
+                other_name = (
+                    community_labels.get(other_cid, f"Community {other_cid}")
+                    if community_labels and other_cid is not None
+                    else f"Community {other_cid}"
+                )
+                other_safe = safe_name(other_name)
+                lines.append(f"- {edge_count} edge{'s' if edge_count != 1 else ''} to [[_COMMUNITY_{other_safe}]]")
+            lines.append("")
+
+        # Top bridge nodes — highest degree nodes that connect to other communities
+        bridge_nodes = [
+            (node_id, G.degree(node_id), _community_reach(node_id))
+            for node_id in members
+            if _community_reach(node_id) > 0
+        ]
+        bridge_nodes.sort(key=lambda x: (-x[2], -x[1]))
+        top_bridges = bridge_nodes[:5]
+        if top_bridges:
+            lines.append("## Top bridge nodes")
+            for node_id, degree, reach in top_bridges:
+                node_label = node_filename[node_id]
+                lines.append(
+                    f"- [[{node_label}]] — degree {degree}, connects to {reach} "
+                    f"{'community' if reach == 1 else 'communities'}"
+                )
+
+        community_safe = safe_name(community_name)
+        fname = f"_COMMUNITY_{community_safe}.md"
+        (out / fname).write_text("\n".join(lines), encoding="utf-8")
+        community_notes_written += 1
+
+    return G.number_of_nodes() + community_notes_written
+
+
+def push_to_neo4j(
+    G: nx.Graph,
+    uri: str,
+    user: str,
+    password: str,
+    communities: dict[int, list[str]] | None = None,
+) -> dict[str, int]:
+    """Push graph directly to a running Neo4j instance via the Python driver.
+
+    Requires: pip install neo4j
+
+    Uses MERGE so re-running is safe — nodes and edges are upserted, not duplicated.
+    Returns a dict with counts of nodes and edges pushed.
+    """
+    try:
+        from neo4j import GraphDatabase
+    except ImportError as e:
+        raise ImportError(
+            "neo4j driver not installed. Run: pip install neo4j"
+        ) from e
+
+    node_community = (
+        {n: cid for cid, nodes in communities.items() for n in nodes}
+        if communities else {}
+    )
+
+    def _safe_rel(relation: str) -> str:
+        return re.sub(r"[^A-Z0-9_]", "_", relation.upper().replace(" ", "_").replace("-", "_")) or "RELATED_TO"
+
+    driver = GraphDatabase.driver(uri, auth=(user, password))
+    nodes_pushed = 0
+    edges_pushed = 0
+
+    with driver.session() as session:
+        for node_id, data in G.nodes(data=True):
+            props = {k: v for k, v in data.items() if isinstance(v, (str, int, float, bool))}
+            props["id"] = node_id
+            cid = node_community.get(node_id)
+            if cid is not None:
+                props["community"] = cid
+            ftype = data.get("file_type", "Entity").capitalize()
+            session.run(
+                f"MERGE (n:{ftype} {{id: $id}}) SET n += $props",
+                id=node_id,
+                props=props,
+            )
+            nodes_pushed += 1
+
+        for u, v, data in G.edges(data=True):
+            rel = _safe_rel(data.get("relation", "RELATED_TO"))
+            props = {k: v for k, v in data.items() if isinstance(v, (str, int, float, bool))}
+            session.run(
+                f"MATCH (a {{id: $src}}), (b {{id: $tgt}}) "
+                f"MERGE (a)-[r:{rel}]->(b) SET r += $props",
+                src=u,
+                tgt=v,
+                props=props,
+            )
+            edges_pushed += 1
+
+    driver.close()
+    return {"nodes": nodes_pushed, "edges": edges_pushed}
+
+
+def to_svg(
+    G: nx.Graph,
+    communities: dict[int, list[str]],
+    output_path: str,
+    community_labels: dict[int, str] | None = None,
+    figsize: tuple[int, int] = (20, 14),
+) -> None:
+    """Export graph as an SVG file using matplotlib + spring layout.
+
+    Lightweight and embeddable — works in Obsidian notes, Notion, GitHub READMEs,
+    and any markdown renderer. No JavaScript required.
+
+    Node size scales with degree. Community colors match the pyvis HTML output.
+    """
+    try:
+        import matplotlib
+        matplotlib.use("Agg")
+        import matplotlib.pyplot as plt
+        import matplotlib.patches as mpatches
+    except ImportError as e:
+        raise ImportError("matplotlib not installed. Run: pip install matplotlib") from e
+
+    node_community = {n: cid for cid, nodes in communities.items() for n in nodes}
+
+    fig, ax = plt.subplots(figsize=figsize, facecolor="#1a1a2e")
+    ax.set_facecolor("#1a1a2e")
+    ax.axis("off")
+
+    pos = nx.spring_layout(G, seed=42, k=2.0 / (G.number_of_nodes() ** 0.5 + 1))
+
+    degree = dict(G.degree())
+    max_deg = max(degree.values()) if degree else 1
+
+    node_colors = [COMMUNITY_COLORS[node_community.get(n, 0) % len(COMMUNITY_COLORS)] for n in G.nodes()]
+    node_sizes = [300 + 1200 * (degree.get(n, 1) / max_deg) for n in G.nodes()]
+
+    # Draw edges — dashed for non-EXTRACTED
+    for u, v, data in G.edges(data=True):
+        conf = data.get("confidence", "EXTRACTED")
+        style = "solid" if conf == "EXTRACTED" else "dashed"
+        alpha = 0.6 if conf == "EXTRACTED" else 0.3
+        x0, y0 = pos[u]
+        x1, y1 = pos[v]
+        ax.plot([x0, x1], [y0, y1], color="#aaaaaa", linewidth=0.8,
+                linestyle=style, alpha=alpha, zorder=1)
+
+    nx.draw_networkx_nodes(G, pos, ax=ax, node_color=node_colors,
+                           node_size=node_sizes, alpha=0.9)
+    nx.draw_networkx_labels(G, pos, ax=ax,
+                            labels={n: G.nodes[n].get("label", n) for n in G.nodes()},
+                            font_size=7, font_color="white")
+
+    # Legend
+    if community_labels:
+        patches = [
+            mpatches.Patch(
+                color=COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)],
+                label=f"{label} ({len(communities.get(cid, []))})",
+            )
+            for cid, label in sorted(community_labels.items())
+        ]
+        ax.legend(handles=patches, loc="upper left", framealpha=0.7,
+                  facecolor="#2a2a4e", labelcolor="white", fontsize=8)
+
+    plt.tight_layout()
+    plt.savefig(output_path, format="svg", bbox_inches="tight",
+                facecolor=fig.get_facecolor())
+    plt.close(fig)

graphify/extract.py

@@ -0,0 +1,343 @@
+"""Deterministic structural extraction from Python code using tree-sitter. Outputs nodes+edges dicts."""
+from __future__ import annotations
+import json
+import re
+import sys
+from pathlib import Path
+
+
+def _make_id(*parts: str) -> str:
+    """Build a stable node ID from one or more name parts."""
+    combined = "_".join(p.strip("_.") for p in parts if p)
+    cleaned = re.sub(r"[^a-zA-Z0-9]+", "_", combined)
+    return cleaned.strip("_").lower()
+
+
+def extract_python(path: Path) -> dict:
+    """Extract classes, functions, and imports from a .py file via tree-sitter AST."""
+    try:
+        import tree_sitter_python as tspython
+        from tree_sitter import Language, Parser
+    except ImportError:
+        return {"nodes": [], "edges": [], "error": "tree-sitter-python not installed"}
+
+    try:
+        language = Language(tspython.language())
+        parser = Parser(language)
+        source = path.read_bytes()
+        tree = parser.parse(source)
+        root = tree.root_node
+    except Exception as e:
+        return {"nodes": [], "edges": [], "error": str(e)}
+
+    stem = path.stem
+    str_path = str(path)
+    nodes: list[dict] = []
+    edges: list[dict] = []
+    seen_ids: set[str] = set()
+
+    def add_node(nid: str, label: str, line: int) -> None:
+        if nid not in seen_ids:
+            seen_ids.add(nid)
+            nodes.append({
+                "id": nid,
+                "label": label,
+                "file_type": "code",
+                "source_file": str_path,
+                "source_location": f"L{line}",
+            })
+
+    def add_edge(src: str, tgt: str, relation: str, line: int) -> None:
+        # Only add edge if both endpoints exist or src is the file node
+        edges.append({
+            "source": src,
+            "target": tgt,
+            "relation": relation,
+            "confidence": "EXTRACTED",
+            "source_file": str_path,
+            "source_location": f"L{line}",
+            "weight": 1.0,
+        })
+
+    # File-level node — stable ID based on stem only
+    file_nid = _make_id(stem)
+    add_node(file_nid, path.name, 1)
+
+    def walk(node, parent_class_nid: str | None = None) -> None:
+        t = node.type
+
+        if t == "import_statement":
+            for child in node.children:
+                if child.type in ("dotted_name", "aliased_import"):
+                    raw = source[child.start_byte:child.end_byte].decode()
+                    module_name = raw.split(" as ")[0].strip().lstrip(".")
+                    tgt_nid = _make_id(module_name)
+                    add_edge(file_nid, tgt_nid, "imports", node.start_point[0] + 1)
+            return
+
+        if t == "import_from_statement":
+            module_node = node.child_by_field_name("module_name")
+            if module_node:
+                raw = source[module_node.start_byte:module_node.end_byte].decode().lstrip(".")
+                tgt_nid = _make_id(raw)
+                add_edge(file_nid, tgt_nid, "imports_from", node.start_point[0] + 1)
+            return
+
+        if t == "class_definition":
+            name_node = node.child_by_field_name("name")
+            if not name_node:
+                return
+            class_name = source[name_node.start_byte:name_node.end_byte].decode()
+            class_nid = _make_id(stem, class_name)
+            line = node.start_point[0] + 1
+            add_node(class_nid, class_name, line)
+            add_edge(file_nid, class_nid, "contains", line)
+
+            # Inheritance — create stub node for external bases so the edge is never dropped
+            args = node.child_by_field_name("superclasses")
+            if args:
+                for arg in args.children:
+                    if arg.type == "identifier":
+                        base = source[arg.start_byte:arg.end_byte].decode()
+                        # Try same-file base first; fall back to a bare stub
+                        base_nid = _make_id(stem, base)
+                        if base_nid not in seen_ids:
+                            # External or forward-declared base — add a stub so edge survives
+                            base_nid = _make_id(base)
+                            if base_nid not in seen_ids:
+                                nodes.append({
+                                    "id": base_nid,
+                                    "label": base,
+                                    "file_type": "code",
+                                    "source_file": "",
+                                    "source_location": "",
+                                })
+                                seen_ids.add(base_nid)
+                        add_edge(class_nid, base_nid, "inherits", line)
+
+            # Walk class body for methods
+            body = node.child_by_field_name("body")
+            if body:
+                for child in body.children:
+                    walk(child, parent_class_nid=class_nid)
+            return
+
+        if t == "function_definition":
+            name_node = node.child_by_field_name("name")
+            if not name_node:
+                return
+            func_name = source[name_node.start_byte:name_node.end_byte].decode()
+            line = node.start_point[0] + 1
+            if parent_class_nid:
+                func_nid = _make_id(parent_class_nid, func_name)
+                add_node(func_nid, f".{func_name}()", line)
+                add_edge(parent_class_nid, func_nid, "method", line)
+            else:
+                func_nid = _make_id(stem, func_name)
+                add_node(func_nid, f"{func_name}()", line)
+                add_edge(file_nid, func_nid, "contains", line)
+            return
+
+        for child in node.children:
+            walk(child, parent_class_nid=None)
+
+    walk(root)
+
+    # Post-process: remove edges whose source or target was never added as a node
+    # (dangling import edges pointing to external libraries are fine to keep,
+    #  but edges between internal entities must be valid)
+    valid_ids = seen_ids
+    clean_edges = []
+    for edge in edges:
+        src, tgt = edge["source"], edge["target"]
+        # Keep if both endpoints are known, OR if it's an import edge (tgt may be external)
+        if src in valid_ids and (tgt in valid_ids or edge["relation"] in ("imports", "imports_from")):
+            clean_edges.append(edge)
+
+    return {"nodes": nodes, "edges": clean_edges}
+
+
+def _resolve_cross_file_imports(
+    per_file: list[dict],
+    paths: list[Path],
+) -> list[dict]:
+    """
+    Two-pass import resolution: turn file-level imports into class-level edges.
+
+    Pass 1 — build a global map: class/function name → node_id, per stem.
+    Pass 2 — for each `from .module import Name`, look up Name in the global
+              map and add a direct INFERRED edge from each class in the
+              importing file to the imported entity.
+
+    This turns:
+        auth.py --imports_from--> models.py          (obvious, filtered out)
+    Into:
+        DigestAuth --uses--> Response  [INFERRED]    (cross-file, interesting!)
+        BasicAuth  --uses--> Request   [INFERRED]
+    """
+    try:
+        import tree_sitter_python as tspython
+        from tree_sitter import Language, Parser
+    except ImportError:
+        return []
+
+    language = Language(tspython.language())
+    parser = Parser(language)
+
+    # Pass 1: name → node_id across all files
+    # Map: stem → {ClassName: node_id}
+    stem_to_entities: dict[str, dict[str, str]] = {}
+    for file_result in per_file:
+        for node in file_result.get("nodes", []):
+            src = node.get("source_file", "")
+            if not src:
+                continue
+            stem = Path(src).stem
+            label = node.get("label", "")
+            nid = node.get("id", "")
+            # Only index real classes/functions (not file nodes, not method stubs)
+            if label and not label.endswith((")", ".py")) and "_" not in label[:1]:
+                stem_to_entities.setdefault(stem, {})[label] = nid
+
+    # Pass 2: for each file, find `from .X import A, B, C` and resolve
+    new_edges: list[dict] = []
+    stem_to_path: dict[str, Path] = {p.stem: p for p in paths}
+
+    for file_result, path in zip(per_file, paths):
+        stem = path.stem
+        str_path = str(path)
+
+        # Find all classes defined in this file (the importers)
+        local_classes = [
+            n["id"] for n in file_result.get("nodes", [])
+            if n.get("source_file") == str_path
+            and not n["label"].endswith((")", ".py"))
+            and n["id"] != _make_id(stem)  # exclude file-level node
+        ]
+        if not local_classes:
+            continue
+
+        # Parse imports from this file
+        try:
+            source = path.read_bytes()
+            tree = parser.parse(source)
+        except Exception:
+            continue
+
+        def walk_imports(node) -> None:
+            if node.type == "import_from_statement":
+                # Find the module name — handles both absolute and relative imports.
+                # Relative: `from .models import X` → relative_import → dotted_name
+                # Absolute: `from models import X`  → module_name field
+                target_stem: str | None = None
+                for child in node.children:
+                    if child.type == "relative_import":
+                        # Dig into relative_import → dotted_name → identifier
+                        for sub in child.children:
+                            if sub.type == "dotted_name":
+                                raw = source[sub.start_byte:sub.end_byte].decode()
+                                target_stem = raw.split(".")[-1]
+                                break
+                        break
+                    if child.type == "dotted_name" and target_stem is None:
+                        raw = source[child.start_byte:child.end_byte].decode()
+                        target_stem = raw.split(".")[-1]
+
+                if not target_stem or target_stem not in stem_to_entities:
+                    return
+
+                # Collect imported names: dotted_name children of import_from_statement
+                # that come AFTER the 'import' keyword token.
+                imported_names: list[str] = []
+                past_import_kw = False
+                for child in node.children:
+                    if child.type == "import":
+                        past_import_kw = True
+                        continue
+                    if not past_import_kw:
+                        continue
+                    if child.type == "dotted_name":
+                        imported_names.append(
+                            source[child.start_byte:child.end_byte].decode()
+                        )
+                    elif child.type == "aliased_import":
+                        # `import X as Y` — take the original name
+                        name_node = child.child_by_field_name("name")
+                        if name_node:
+                            imported_names.append(
+                                source[name_node.start_byte:name_node.end_byte].decode()
+                            )
+
+                line = node.start_point[0] + 1
+                for name in imported_names:
+                    tgt_nid = stem_to_entities[target_stem].get(name)
+                    if tgt_nid:
+                        for src_class_nid in local_classes:
+                            new_edges.append({
+                                "source": src_class_nid,
+                                "target": tgt_nid,
+                                "relation": "uses",
+                                "confidence": "INFERRED",
+                                "source_file": str_path,
+                                "source_location": f"L{line}",
+                                "weight": 0.8,
+                            })
+            for child in node.children:
+                walk_imports(child)
+
+        walk_imports(tree.root_node)
+
+    return new_edges
+
+
+def extract(paths: list[Path]) -> dict:
+    """Extract AST nodes and edges from a list of code files.
+
+    Two-pass process:
+    1. Per-file structural extraction (classes, functions, imports)
+    2. Cross-file import resolution: turns file-level imports into
+       class-level INFERRED edges (DigestAuth --uses--> Response)
+    """
+    per_file: list[dict] = []
+
+    for path in paths:
+        if path.suffix == ".py":
+            result = extract_python(path)
+            per_file.append(result)
+
+    all_nodes: list[dict] = []
+    all_edges: list[dict] = []
+    for result in per_file:
+        all_nodes.extend(result.get("nodes", []))
+        all_edges.extend(result.get("edges", []))
+
+    # Add cross-file class-level edges
+    cross_file_edges = _resolve_cross_file_imports(per_file, paths)
+    all_edges.extend(cross_file_edges)
+
+    return {
+        "nodes": all_nodes,
+        "edges": all_edges,
+        "input_tokens": 0,
+        "output_tokens": 0,
+    }
+
+
+def collect_files(target: Path) -> list[Path]:
+    if target.is_file():
+        return [target]
+    return sorted(p for p in target.rglob("*.py")
+                  if not any(part.startswith(".") for part in p.parts))
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: python -m graphify.extract <file_or_dir> ...", file=sys.stderr)
+        sys.exit(1)
+
+    paths: list[Path] = []
+    for arg in sys.argv[1:]:
+        paths.extend(collect_files(Path(arg)))
+
+    result = extract(paths)
+    print(json.dumps(result, indent=2))

graphify/ingest.py

@@ -0,0 +1,233 @@
+# fetch URLs (tweet/arxiv/pdf/web) and save as annotated markdown
+from __future__ import annotations
+import json
+import re
+import sys
+import urllib.request
+import urllib.parse
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+def _safe_filename(url: str, suffix: str) -> str:
+    """Turn a URL into a safe filename."""
+    parsed = urllib.parse.urlparse(url)
+    name = parsed.netloc + parsed.path
+    name = re.sub(r"[^\w\-]", "_", name).strip("_")
+    name = re.sub(r"_+", "_", name)[:80]
+    return name + suffix
+
+
+def _detect_url_type(url: str) -> str:
+    """Classify the URL for targeted extraction."""
+    lower = url.lower()
+    if "twitter.com" in lower or "x.com" in lower:
+        return "tweet"
+    if "arxiv.org" in lower:
+        return "arxiv"
+    if "github.com" in lower:
+        return "github"
+    if "youtube.com" in lower or "youtu.be" in lower:
+        return "youtube"
+    parsed = urllib.parse.urlparse(url)
+    path = parsed.path.lower()
+    if path.endswith(".pdf"):
+        return "pdf"
+    if any(path.endswith(ext) for ext in (".png", ".jpg", ".jpeg", ".webp", ".gif")):
+        return "image"
+    return "webpage"
+
+
+def _fetch_html(url: str) -> str:
+    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 graphify/1.0"})
+    with urllib.request.urlopen(req, timeout=15) as resp:
+        return resp.read().decode("utf-8", errors="ignore")
+
+
+def _html_to_markdown(html: str, url: str) -> str:
+    """Convert HTML to clean markdown. Uses html2text if available, else basic strip."""
+    try:
+        import html2text
+        h = html2text.HTML2Text()
+        h.ignore_links = False
+        h.ignore_images = True
+        h.body_width = 0
+        return h.handle(html)
+    except ImportError:
+        # Fallback: strip tags
+        text = re.sub(r"<script[^>]*>.*?</script>", "", html, flags=re.DOTALL | re.IGNORECASE)
+        text = re.sub(r"<style[^>]*>.*?</style>", "", text, flags=re.DOTALL | re.IGNORECASE)
+        text = re.sub(r"<[^>]+>", " ", text)
+        text = re.sub(r"\s+", " ", text).strip()
+        return text[:8000]
+
+
+def _fetch_tweet(url: str, author: str | None, contributor: str | None) -> tuple[str, str]:
+    """Fetch a tweet URL. Returns (content, filename)."""
+    # Normalize to twitter.com for oEmbed
+    oembed_url = url.replace("x.com", "twitter.com")
+    oembed_api = f"https://publish.twitter.com/oembed?url={urllib.parse.quote(oembed_url)}&omit_script=true"
+    try:
+        req = urllib.request.Request(oembed_api, headers={"User-Agent": "graphify/1.0"})
+        with urllib.request.urlopen(req, timeout=10) as resp:
+            data = json.loads(resp.read())
+        tweet_text = re.sub(r"<[^>]+>", "", data.get("html", "")).strip()
+        tweet_author = data.get("author_name", "unknown")
+    except Exception:
+        # oEmbed failed — save URL stub
+        tweet_text = f"Tweet at {url} (could not fetch content)"
+        tweet_author = "unknown"
+
+    now = datetime.now(timezone.utc).isoformat()
+    content = f"""---
+source_url: {url}
+type: tweet
+author: {tweet_author}
+captured_at: {now}
+contributor: {contributor or author or 'unknown'}
+---
+
+# Tweet by @{tweet_author}
+
+{tweet_text}
+
+Source: {url}
+"""
+    filename = _safe_filename(url, ".md")
+    return content, filename
+
+
+def _fetch_webpage(url: str, author: str | None, contributor: str | None) -> tuple[str, str]:
+    """Fetch a generic webpage and convert to markdown."""
+    html = _fetch_html(url)
+    # Extract title
+    title_match = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
+    title = re.sub(r"\s+", " ", title_match.group(1)).strip() if title_match else url
+
+    markdown = _html_to_markdown(html, url)
+    now = datetime.now(timezone.utc).isoformat()
+    content = f"""---
+source_url: {url}
+type: webpage
+title: "{title}"
+captured_at: {now}
+contributor: {contributor or author or 'unknown'}
+---
+
+# {title}
+
+Source: {url}
+
+---
+
+{markdown[:12000]}
+"""
+    filename = _safe_filename(url, ".md")
+    return content, filename
+
+
+def _fetch_arxiv(url: str, author: str | None, contributor: str | None) -> tuple[str, str]:
+    """Fetch arXiv abstract page."""
+    # Convert /abs/ or /pdf/ to abs for the API
+    arxiv_id = re.search(r"(\d{4}\.\d{4,5})", url)
+    if arxiv_id:
+        api_url = f"https://export.arxiv.org/abs/{arxiv_id.group(1)}"
+        try:
+            html = _fetch_html(api_url)
+            abstract_match = re.search(r'class="abstract[^"]*"[^>]*>(.*?)</blockquote>', html, re.DOTALL | re.IGNORECASE)
+            abstract = re.sub(r"<[^>]+>", "", abstract_match.group(1)).strip() if abstract_match else ""
+            title_match = re.search(r'class="title[^"]*"[^>]*>(.*?)</h1>', html, re.DOTALL | re.IGNORECASE)
+            title = re.sub(r"<[^>]+>", " ", title_match.group(1)).strip() if title_match else arxiv_id.group(1)
+            authors_match = re.search(r'class="authors"[^>]*>(.*?)</div>', html, re.DOTALL | re.IGNORECASE)
+            paper_authors = re.sub(r"<[^>]+>", "", authors_match.group(1)).strip() if authors_match else ""
+        except Exception:
+            title, abstract, paper_authors = arxiv_id.group(1), "", ""
+    else:
+        return _fetch_webpage(url, author, contributor)
+
+    now = datetime.now(timezone.utc).isoformat()
+    content = f"""---
+source_url: {url}
+arxiv_id: {arxiv_id.group(1) if arxiv_id else ''}
+type: paper
+title: "{title}"
+paper_authors: "{paper_authors}"
+captured_at: {now}
+contributor: {contributor or author or 'unknown'}
+---
+
+# {title}
+
+**Authors:** {paper_authors}
+**arXiv:** {arxiv_id.group(1) if arxiv_id else url}
+
+## Abstract
+
+{abstract}
+
+Source: {url}
+"""
+    filename = f"arxiv_{arxiv_id.group(1).replace('.', '_')}.md" if arxiv_id else _safe_filename(url, ".md")
+    return content, filename
+
+
+def _download_binary(url: str, suffix: str, target_dir: Path) -> Path:
+    """Download a binary file (PDF, image) directly."""
+    filename = _safe_filename(url, suffix)
+    out_path = target_dir / filename
+    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 graphify/1.0"})
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        out_path.write_bytes(resp.read())
+    return out_path
+
+
+def ingest(url: str, target_dir: Path, author: str | None = None, contributor: str | None = None) -> Path:
+    """
+    Fetch a URL and save it into target_dir as a graphify-ready file.
+
+    Returns the path of the saved file.
+    """
+    target_dir.mkdir(parents=True, exist_ok=True)
+    url_type = _detect_url_type(url)
+
+    if url_type == "pdf":
+        out = _download_binary(url, ".pdf", target_dir)
+        print(f"Downloaded PDF: {out.name}")
+        return out
+
+    if url_type == "image":
+        suffix = Path(urllib.parse.urlparse(url).path).suffix or ".jpg"
+        out = _download_binary(url, suffix, target_dir)
+        print(f"Downloaded image: {out.name}")
+        return out
+
+    if url_type == "tweet":
+        content, filename = _fetch_tweet(url, author, contributor)
+    elif url_type == "arxiv":
+        content, filename = _fetch_arxiv(url, author, contributor)
+    else:
+        content, filename = _fetch_webpage(url, author, contributor)
+
+    out_path = target_dir / filename
+    # Avoid overwriting — append counter if needed
+    counter = 1
+    while out_path.exists():
+        stem = Path(filename).stem
+        out_path = target_dir / f"{stem}_{counter}.md"
+        counter += 1
+
+    out_path.write_text(content, encoding="utf-8")
+    print(f"Saved {url_type}: {out_path.name}")
+    return out_path
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description="Fetch a URL into a graphify /raw folder")
+    parser.add_argument("url", help="URL to fetch")
+    parser.add_argument("target_dir", nargs="?", default="./raw", help="Target directory (default: ./raw)")
+    parser.add_argument("--author", help="Your name (stored as node metadata)")
+    parser.add_argument("--contributor", help="Contributor name for team graphs")
+    args = parser.parse_args()
+    out = ingest(args.url, Path(args.target_dir), author=args.author, contributor=args.contributor)
+    print(f"Ready for graphify: {out}")

graphify/report.py

@@ -0,0 +1,128 @@
+# generate GRAPH_REPORT.md — the human-readable audit trail
+from __future__ import annotations
+from datetime import date
+import networkx as nx
+
+
+def generate(
+    G: nx.Graph,
+    communities: dict[int, list[str]],
+    cohesion_scores: dict[int, float],
+    community_labels: dict[int, str],
+    god_node_list: list[dict],
+    surprise_list: list[dict],
+    detection_result: dict,
+    token_cost: dict,
+    root: str,
+    suggested_questions: list[dict] | None = None,
+) -> str:
+    today = date.today().isoformat()
+
+    confidences = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)]
+    total = len(confidences) or 1
+    ext_pct = round(confidences.count("EXTRACTED") / total * 100)
+    inf_pct = round(confidences.count("INFERRED") / total * 100)
+    amb_pct = round(confidences.count("AMBIGUOUS") / total * 100)
+
+    lines = [
+        f"# Graph Report — {root}  ({today})",
+        "",
+        "## Corpus Check",
+    ]
+    if detection_result.get("warning"):
+        lines.append(f"- {detection_result['warning']}")
+    else:
+        lines += [
+            f"- {detection_result['total_files']} files · ~{detection_result['total_words']:,} words",
+            "- Verdict: corpus is large enough that graph structure adds value.",
+        ]
+
+    lines += [
+        "",
+        "## Summary",
+        f"- {G.number_of_nodes()} nodes · {G.number_of_edges()} edges · {len(communities)} communities detected",
+        f"- Extraction: {ext_pct}% EXTRACTED · {inf_pct}% INFERRED · {amb_pct}% AMBIGUOUS",
+        f"- Token cost: {token_cost.get('input', 0):,} input · {token_cost.get('output', 0):,} output",
+        "",
+        "## God Nodes (most connected — your core abstractions)",
+    ]
+    for i, node in enumerate(god_node_list, 1):
+        lines.append(f"{i}. `{node['label']}` — {node['edges']} edges")
+
+    lines += ["", "## Surprising Connections (you probably didn't know these)"]
+    if surprise_list:
+        for s in surprise_list:
+            relation = s.get("relation", "related_to")
+            note = s.get("note", "")
+            files = s.get("source_files", ["", ""])
+            lines += [
+                f"- `{s['source']}` --{relation}--> `{s['target']}`  [{s['confidence']}]",
+                f"  {files[0]} → {files[1]}" + (f"  _{note}_" if note else ""),
+            ]
+    else:
+        lines.append("- None detected — all connections are within the same source files.")
+
+    lines += ["", "## Communities"]
+    from .analyze import _is_file_node as _ifn
+    for cid, nodes in communities.items():
+        label = community_labels.get(cid, f"Community {cid}")
+        score = cohesion_scores.get(cid, 0.0)
+        # Filter method/function stubs from display — they're structural noise
+        real_nodes = [n for n in nodes if not _ifn(G, n)]
+        display = [G.nodes[n].get("label", n) for n in real_nodes[:8]]
+        suffix = f" (+{len(real_nodes)-8} more)" if len(real_nodes) > 8 else ""
+        lines += [
+            "",
+            f"### Community {cid} — \"{label}\"",
+            f"Cohesion: {score}",
+            f"Nodes ({len(real_nodes)}): {', '.join(display)}{suffix}",
+        ]
+
+    ambiguous = [(u, v, d) for u, v, d in G.edges(data=True) if d.get("confidence") == "AMBIGUOUS"]
+    if ambiguous:
+        lines += ["", "## Ambiguous Edges — Review These"]
+        for u, v, d in ambiguous:
+            ul = G.nodes[u].get("label", u)
+            vl = G.nodes[v].get("label", v)
+            lines += [
+                f"- `{ul}` → `{vl}`  [AMBIGUOUS]",
+                f"  {d.get('source_file', '')} · relation: {d.get('relation', 'unknown')}",
+            ]
+
+    # --- Gaps section ---
+    from .analyze import _is_file_node, _is_concept_node
+
+    isolated = [
+        n for n in G.nodes()
+        if G.degree(n) <= 1 and not _is_file_node(G, n) and not _is_concept_node(G, n)
+    ]
+    thin_communities = {
+        cid: nodes for cid, nodes in communities.items() if len(nodes) < 3
+    }
+    gap_count = len(isolated) + len(thin_communities)
+
+    if gap_count > 0 or amb_pct > 20:
+        lines += ["", "## Knowledge Gaps"]
+        if isolated:
+            isolated_labels = [G.nodes[n].get("label", n) for n in isolated[:5]]
+            suffix = f" (+{len(isolated)-5} more)" if len(isolated) > 5 else ""
+            lines.append(f"- **{len(isolated)} isolated node(s):** {', '.join(f'`{l}`' for l in isolated_labels)}{suffix}")
+            lines.append("  These have ≤1 connection — possible missing edges or undocumented components.")
+        if thin_communities:
+            for cid, nodes in thin_communities.items():
+                label = community_labels.get(cid, f"Community {cid}")
+                node_labels = [G.nodes[n].get("label", n) for n in nodes]
+                lines.append(f"- **Thin community `{label}`** ({len(nodes)} nodes): {', '.join(f'`{l}`' for l in node_labels)}")
+                lines.append("  Too small to be a meaningful cluster — may be noise or needs more connections extracted.")
+        if amb_pct > 20:
+            lines.append(f"- **High ambiguity: {amb_pct}% of edges are AMBIGUOUS.** Review the Ambiguous Edges section above.")
+
+    if suggested_questions:
+        lines += ["", "## Suggested Questions"]
+        lines.append("_Questions this graph is uniquely positioned to answer:_")
+        lines.append("")
+        for q in suggested_questions:
+            lines.append(f"- **{q['question']}**")
+            lines.append(f"  _{q['why']}_")
+
+    return "\n".join(lines)

graphify/serve.py

@@ -0,0 +1,270 @@
+# MCP stdio server — exposes graph query tools to Claude and other agents
+from __future__ import annotations
+import json
+import sys
+from pathlib import Path
+import networkx as nx
+from networkx.readwrite import json_graph
+
+
+def _load_graph(graph_path: str) -> nx.Graph:
+    data = json.loads(Path(graph_path).read_text())
+    return json_graph.node_link_graph(data, edges="links")
+
+
+def _communities_from_graph(G: nx.Graph) -> dict[int, list[str]]:
+    """Reconstruct community dict from community property stored on nodes."""
+    communities: dict[int, list[str]] = {}
+    for node_id, data in G.nodes(data=True):
+        cid = data.get("community")
+        if cid is not None:
+            communities.setdefault(int(cid), []).append(node_id)
+    return communities
+
+
+def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]:
+    scored = []
+    for nid, data in G.nodes(data=True):
+        label = data.get("label", "").lower()
+        source = data.get("source_file", "").lower()
+        score = sum(1 for t in terms if t in label) + sum(0.5 for t in terms if t in source)
+        if score > 0:
+            scored.append((score, nid))
+    return sorted(scored, reverse=True)
+
+
+def _bfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], list[tuple]]:
+    visited: set[str] = set(start_nodes)
+    frontier = set(start_nodes)
+    edges_seen: list[tuple] = []
+    for _ in range(depth):
+        next_frontier: set[str] = set()
+        for n in frontier:
+            for neighbor in G.neighbors(n):
+                if neighbor not in visited:
+                    next_frontier.add(neighbor)
+                    edges_seen.append((n, neighbor))
+        visited.update(next_frontier)
+        frontier = next_frontier
+    return visited, edges_seen
+
+
+def _dfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], list[tuple]]:
+    visited: set[str] = set()
+    edges_seen: list[tuple] = []
+    stack = [(n, 0) for n in reversed(start_nodes)]
+    while stack:
+        node, d = stack.pop()
+        if node in visited or d > depth:
+            continue
+        visited.add(node)
+        for neighbor in G.neighbors(node):
+            if neighbor not in visited:
+                stack.append((neighbor, d + 1))
+                edges_seen.append((node, neighbor))
+    return visited, edges_seen
+
+
+def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_budget: int = 2000) -> str:
+    """Render subgraph as text, cutting at token_budget (approx 4 chars/token)."""
+    char_budget = token_budget * 4
+    lines = []
+    for nid in sorted(nodes, key=lambda n: G.degree(n), reverse=True):
+        d = G.nodes[nid]
+        line = f"NODE {d.get('label', nid)} [src={d.get('source_file', '')} loc={d.get('source_location', '')} community={d.get('community', '')}]"
+        lines.append(line)
+    for u, v in edges:
+        if u in nodes and v in nodes:
+            d = G.edges[u, v]
+            line = f"EDGE {G.nodes[u].get('label', u)} --{d.get('relation', '')} [{d.get('confidence', '')}]--> {G.nodes[v].get('label', v)}"
+            lines.append(line)
+    output = "\n".join(lines)
+    if len(output) > char_budget:
+        output = output[:char_budget] + f"\n... (truncated to ~{token_budget} token budget)"
+    return output
+
+
+def serve(graph_path: str = ".graphify/graph.json") -> None:
+    """Start the MCP server. Requires pip install mcp."""
+    try:
+        from mcp.server import Server
+        from mcp.server.stdio import stdio_server
+        from mcp import types
+    except ImportError as e:
+        raise ImportError("mcp not installed. Run: pip install mcp") from e
+
+    G = _load_graph(graph_path)
+    communities = _communities_from_graph(G)
+
+    server = Server("graphify")
+
+    @server.list_tools()
+    async def list_tools() -> list[types.Tool]:
+        return [
+            types.Tool(
+                name="query_graph",
+                description="Search the knowledge graph using BFS or DFS. Returns relevant nodes and edges as text context.",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "question": {"type": "string", "description": "Natural language question or keyword search"},
+                        "mode": {"type": "string", "enum": ["bfs", "dfs"], "default": "bfs",
+                                 "description": "bfs=broad context, dfs=trace a specific path"},
+                        "depth": {"type": "integer", "default": 3, "description": "Traversal depth (1-6)"},
+                        "token_budget": {"type": "integer", "default": 2000, "description": "Max output tokens"},
+                    },
+                    "required": ["question"],
+                },
+            ),
+            types.Tool(
+                name="get_node",
+                description="Get full details for a specific node by label or ID.",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "label": {"type": "string", "description": "Node label or ID to look up"},
+                    },
+                    "required": ["label"],
+                },
+            ),
+            types.Tool(
+                name="get_neighbors",
+                description="Get all direct neighbors of a node with edge details.",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "label": {"type": "string"},
+                        "relation_filter": {"type": "string", "description": "Optional: filter by relation type"},
+                    },
+                    "required": ["label"],
+                },
+            ),
+            types.Tool(
+                name="get_community",
+                description="Get all nodes in a community by community ID or label.",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "community_id": {"type": "integer", "description": "Community ID (0-indexed by size)"},
+                    },
+                    "required": ["community_id"],
+                },
+            ),
+            types.Tool(
+                name="god_nodes",
+                description="Return the most connected nodes — the core abstractions of the knowledge graph.",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "top_n": {"type": "integer", "default": 10},
+                    },
+                },
+            ),
+            types.Tool(
+                name="graph_stats",
+                description="Return summary statistics: node count, edge count, communities, confidence breakdown.",
+                inputSchema={"type": "object", "properties": {}},
+            ),
+        ]
+
+    @server.call_tool()
+    async def call_tool(name: str, arguments: dict) -> list[types.TextContent]:
+        if name == "query_graph":
+            question = arguments["question"]
+            mode = arguments.get("mode", "bfs")
+            depth = min(int(arguments.get("depth", 3)), 6)
+            budget = int(arguments.get("token_budget", 2000))
+            terms = [t.lower() for t in question.split() if len(t) > 2]
+            scored = _score_nodes(G, terms)
+            start_nodes = [nid for _, nid in scored[:3]]
+            if not start_nodes:
+                return [types.TextContent(type="text", text="No matching nodes found.")]
+            if mode == "dfs":
+                nodes, edges = _dfs(G, start_nodes, depth)
+            else:
+                nodes, edges = _bfs(G, start_nodes, depth)
+            text = f"Traversal: {mode.upper()} depth={depth} | Start: {[G.nodes[n].get('label', n) for n in start_nodes]} | {len(nodes)} nodes found\n\n"
+            text += _subgraph_to_text(G, nodes, edges, budget)
+            return [types.TextContent(type="text", text=text)]
+
+        elif name == "get_node":
+            label = arguments["label"].lower()
+            matches = [(nid, d) for nid, d in G.nodes(data=True)
+                       if label in d.get("label", "").lower() or label == nid.lower()]
+            if not matches:
+                return [types.TextContent(type="text", text=f"No node matching '{label}' found.")]
+            nid, d = matches[0]
+            lines = [f"Node: {d.get('label', nid)}",
+                     f"  ID: {nid}",
+                     f"  Source: {d.get('source_file', '')} {d.get('source_location', '')}",
+                     f"  Type: {d.get('file_type', '')}",
+                     f"  Community: {d.get('community', '')}",
+                     f"  Degree: {G.degree(nid)}"]
+            return [types.TextContent(type="text", text="\n".join(lines))]
+
+        elif name == "get_neighbors":
+            label = arguments["label"].lower()
+            rel_filter = arguments.get("relation_filter", "").lower()
+            matches = [nid for nid, d in G.nodes(data=True)
+                       if label in d.get("label", "").lower() or label == nid.lower()]
+            if not matches:
+                return [types.TextContent(type="text", text=f"No node matching '{label}' found.")]
+            nid = matches[0]
+            lines = [f"Neighbors of {G.nodes[nid].get('label', nid)}:"]
+            for neighbor in G.neighbors(nid):
+                d = G.edges[nid, neighbor]
+                rel = d.get("relation", "")
+                if rel_filter and rel_filter not in rel.lower():
+                    continue
+                conf = d.get("confidence", "")
+                nlabel = G.nodes[neighbor].get("label", neighbor)
+                lines.append(f"  --> {nlabel} [{rel}] [{conf}]")
+            return [types.TextContent(type="text", text="\n".join(lines))]
+
+        elif name == "get_community":
+            cid = int(arguments["community_id"])
+            nodes = communities.get(cid, [])
+            if not nodes:
+                return [types.TextContent(type="text", text=f"Community {cid} not found.")]
+            lines = [f"Community {cid} ({len(nodes)} nodes):"]
+            for n in nodes:
+                d = G.nodes[n]
+                lines.append(f"  {d.get('label', n)} [{d.get('source_file', '')}]")
+            return [types.TextContent(type="text", text="\n".join(lines))]
+
+        elif name == "god_nodes":
+            from .analyze import god_nodes as _god_nodes
+            top_n = int(arguments.get("top_n", 10))
+            nodes = _god_nodes(G, top_n=top_n)
+            lines = ["God nodes (most connected):"]
+            for i, n in enumerate(nodes, 1):
+                lines.append(f"  {i}. {n['label']} — {n['edges']} edges")
+            return [types.TextContent(type="text", text="\n".join(lines))]
+
+        elif name == "graph_stats":
+            confs = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)]
+            total = len(confs) or 1
+            text = (
+                f"Nodes: {G.number_of_nodes()}\n"
+                f"Edges: {G.number_of_edges()}\n"
+                f"Communities: {len(communities)}\n"
+                f"EXTRACTED: {round(confs.count('EXTRACTED')/total*100)}%\n"
+                f"INFERRED: {round(confs.count('INFERRED')/total*100)}%\n"
+                f"AMBIGUOUS: {round(confs.count('AMBIGUOUS')/total*100)}%\n"
+            )
+            return [types.TextContent(type="text", text=text)]
+
+        return [types.TextContent(type="text", text=f"Unknown tool: {name}")]
+
+    import asyncio
+
+    async def main() -> None:
+        async with stdio_server() as streams:
+            await server.run(streams[0], streams[1], server.create_initialization_options())
+
+    asyncio.run(main())
+
+
+if __name__ == "__main__":
+    graph_path = sys.argv[1] if len(sys.argv) > 1 else ".graphify/graph.json"
+    serve(graph_path)

graphify/validate.py

@@ -0,0 +1,71 @@
+# validate extraction JSON against the graphify schema before graph assembly
+from __future__ import annotations
+
+VALID_FILE_TYPES = {"code", "document", "paper"}
+VALID_CONFIDENCES = {"EXTRACTED", "INFERRED", "AMBIGUOUS"}
+REQUIRED_NODE_FIELDS = {"id", "label", "file_type", "source_file"}
+REQUIRED_EDGE_FIELDS = {"source", "target", "relation", "confidence", "source_file"}
+
+
+def validate_extraction(data: dict) -> list[str]:
+    """
+    Validate an extraction JSON dict against the graphify schema.
+    Returns a list of error strings — empty list means valid.
+    """
+    if not isinstance(data, dict):
+        return ["Extraction must be a JSON object"]
+
+    errors: list[str] = []
+
+    # Nodes
+    if "nodes" not in data:
+        errors.append("Missing required key 'nodes'")
+    elif not isinstance(data["nodes"], list):
+        errors.append("'nodes' must be a list")
+    else:
+        for i, node in enumerate(data["nodes"]):
+            if not isinstance(node, dict):
+                errors.append(f"Node {i} must be an object")
+                continue
+            for field in REQUIRED_NODE_FIELDS:
+                if field not in node:
+                    errors.append(f"Node {i} (id={node.get('id', '?')!r}) missing required field '{field}'")
+            if "file_type" in node and node["file_type"] not in VALID_FILE_TYPES:
+                errors.append(
+                    f"Node {i} (id={node.get('id', '?')!r}) has invalid file_type "
+                    f"'{node['file_type']}' — must be one of {sorted(VALID_FILE_TYPES)}"
+                )
+
+    # Edges
+    if "edges" not in data:
+        errors.append("Missing required key 'edges'")
+    elif not isinstance(data["edges"], list):
+        errors.append("'edges' must be a list")
+    else:
+        node_ids = {n["id"] for n in data.get("nodes", []) if isinstance(n, dict) and "id" in n}
+        for i, edge in enumerate(data["edges"]):
+            if not isinstance(edge, dict):
+                errors.append(f"Edge {i} must be an object")
+                continue
+            for field in REQUIRED_EDGE_FIELDS:
+                if field not in edge:
+                    errors.append(f"Edge {i} missing required field '{field}'")
+            if "confidence" in edge and edge["confidence"] not in VALID_CONFIDENCES:
+                errors.append(
+                    f"Edge {i} has invalid confidence '{edge['confidence']}' "
+                    f"— must be one of {sorted(VALID_CONFIDENCES)}"
+                )
+            if "source" in edge and node_ids and edge["source"] not in node_ids:
+                errors.append(f"Edge {i} source '{edge['source']}' does not match any node id")
+            if "target" in edge and node_ids and edge["target"] not in node_ids:
+                errors.append(f"Edge {i} target '{edge['target']}' does not match any node id")
+
+    return errors
+
+
+def assert_valid(data: dict) -> None:
+    """Raise ValueError with all errors if extraction is invalid."""
+    errors = validate_extraction(data)
+    if errors:
+        msg = f"Extraction JSON has {len(errors)} error(s):\n" + "\n".join(f"  • {e}" for e in errors)
+        raise ValueError(msg)

graphify/watch.py

@@ -0,0 +1,81 @@
+# monitor a folder and auto-trigger --update when files change
+from __future__ import annotations
+import time
+from pathlib import Path
+
+
+_WATCHED_EXTENSIONS = {
+    ".py", ".ts", ".js", ".go", ".rs", ".java", ".cpp", ".c", ".rb", ".swift", ".kt",
+    ".md", ".txt", ".rst", ".pdf",
+    ".png", ".jpg", ".jpeg", ".webp", ".gif", ".svg",
+}
+
+
+def _run_update(watch_path: Path) -> None:
+    """Write a flag file and print a notification when files change."""
+    flag = watch_path / ".graphify" / "needs_update"
+    flag.parent.mkdir(parents=True, exist_ok=True)
+    flag.write_text("1")
+    print(f"\n[graphify watch] New or changed files detected in {watch_path}")
+    print("[graphify watch] Run `/graphify --update` in Claude Code to update the graph.")
+    print(f"[graphify watch] Flag written to {flag}")
+
+
+def watch(watch_path: Path, debounce: float = 3.0) -> None:
+    """
+    Watch watch_path for new or modified files and re-run graphify --update.
+
+    debounce: seconds to wait after the last change before triggering (avoids
+    running on every keystroke when many files are saved at once).
+    """
+    try:
+        from watchdog.observers import Observer
+        from watchdog.events import FileSystemEventHandler
+    except ImportError as e:
+        raise ImportError("watchdog not installed. Run: pip install watchdog") from e
+
+    last_trigger: float = 0.0
+    pending: bool = False
+
+    class Handler(FileSystemEventHandler):
+        def on_any_event(self, event):
+            nonlocal last_trigger, pending
+            if event.is_directory:
+                return
+            path = Path(event.src_path)
+            if path.suffix.lower() not in _WATCHED_EXTENSIONS:
+                return
+            if any(part.startswith(".") for part in path.parts):
+                return
+            last_trigger = time.monotonic()
+            pending = True
+
+    handler = Handler()
+    observer = Observer()
+    observer.schedule(handler, str(watch_path), recursive=True)
+    observer.start()
+
+    print(f"[graphify watch] Watching {watch_path.resolve()} — press Ctrl+C to stop")
+    print(f"[graphify watch] Debounce: {debounce}s — will update {debounce}s after last change")
+
+    try:
+        while True:
+            time.sleep(0.5)
+            if pending and (time.monotonic() - last_trigger) >= debounce:
+                pending = False
+                _run_update(watch_path)
+    except KeyboardInterrupt:
+        print("\n[graphify watch] Stopped.")
+    finally:
+        observer.stop()
+        observer.join()
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description="Watch a folder and auto-update the graphify graph")
+    parser.add_argument("path", nargs="?", default=".", help="Folder to watch (default: .)")
+    parser.add_argument("--debounce", type=float, default=3.0,
+                        help="Seconds to wait after last change before updating (default: 3)")
+    args = parser.parse_args()
+    watch(Path(args.path), debounce=args.debounce)