reduce graph update churn and stabilize community IDs

Make `graphify update` idempotent by skipping output rewrites when graph/report content is unchanged, add `update --no-cluster`, and preserve community IDs across runs via overlap-based remapping with deterministic partition inputs. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-21 21:16:31 +00:00 · 2026-05-12 02:23:51 +03:00
parent 094d8ba731
commit 8e4c803f31
6 changed files with 320 additions and 37 deletions
@@ -1151,6 +1151,7 @@ def main() -> None:
        print("  update <path>           re-extract code files and update the graph (no LLM needed)")
        print("    --force                 overwrite graph.json even if the rebuild has fewer nodes")
        print("                            (also: GRAPHIFY_FORCE=1 env var; use after refactors that delete code)")
+        print("    --no-cluster            skip clustering, write raw extraction only")
        print("  cluster-only <path>     rerun clustering on an existing graph.json and regenerate report")
        print("    --no-viz                skip graph.html generation (useful for >5000 node graphs / CI)")
        print("    --graph <path>          path to graph.json (default <path>/graphify-out/graph.json)")
@@ -1717,12 +1718,26 @@ def main() -> None:

    elif cmd == "update":
        force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes")
-        argv = list(sys.argv)
-        if "--force" in argv[2:]:
-            force = True
-            argv = [a for a in argv if a != "--force"]
-        if len(argv) > 2:
-            watch_path = Path(argv[2])
+        no_cluster = False
+        args = sys.argv[2:]
+        watch_arg: str | None = None
+        for a in args:
+            if a == "--force":
+                force = True
+                continue
+            if a == "--no-cluster":
+                no_cluster = True
+                continue
+            if a.startswith("-"):
+                print(f"error: unknown update option: {a}", file=sys.stderr)
+                sys.exit(2)
+            if watch_arg is not None:
+                print("error: update accepts at most one path argument", file=sys.stderr)
+                sys.exit(2)
+            watch_arg = a
+
+        if watch_arg is not None:
+            watch_path = Path(watch_arg)
        else:
            # Try to recover the scan root saved by the last full build
            saved = Path(_GRAPHIFY_OUT) / ".graphify_root"
@@ -1738,7 +1753,7 @@ def main() -> None:
        # Interactive CLI: block on the per-repo lock rather than skip, so the
        # user sees their explicit `graphify update` complete instead of
        # exiting silently when a hook-driven rebuild happens to be running.
-        ok = _rebuild_code(watch_path, force=force, block_on_lock=True)
+        ok = _rebuild_code(watch_path, force=force, no_cluster=no_cluster, block_on_lock=True)
        if ok:
            print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.")
            if not (
@@ -3,6 +3,7 @@ from __future__ import annotations
 import contextlib
 import inspect
 import io
+import json
 import sys
 import networkx as nx

@@ -27,15 +28,30 @@ def _partition(G: nx.Graph) -> dict[str, int]:
    Output from graspologic is suppressed to prevent ANSI escape codes
    from corrupting terminal scroll buffers on Windows PowerShell 5.1.
    """
+    stable = nx.Graph()
+    stable.add_nodes_from(sorted(G.nodes(), key=str))
+    edge_rows = sorted(
+        G.edges(data=True),
+        key=lambda row: (str(row[0]), str(row[1]), json.dumps(row[2], sort_keys=True, ensure_ascii=False)),
+    )
+    for src, tgt, attrs in edge_rows:
+        stable.add_edge(src, tgt, **attrs)
+
    try:
        from graspologic.partition import leiden
+        lsig = inspect.signature(leiden).parameters
+        kwargs: dict = {}
+        if "random_seed" in lsig:
+            kwargs["random_seed"] = 42
+        if "trials" in lsig:
+            kwargs["trials"] = 1
        # Suppress graspologic output to prevent ANSI escape codes from
        # corrupting PowerShell 5.1 scroll buffer (issue #19)
        old_stderr = sys.stderr
        try:
            sys.stderr = io.StringIO()
            with _suppress_output():
-                result = leiden(G)
+                result = leiden(stable, **kwargs)
        finally:
            sys.stderr = old_stderr
        return result
@@ -48,7 +64,7 @@ def _partition(G: nx.Graph) -> dict[str, int]:
    kwargs: dict = {"seed": 42, "threshold": 1e-4}
    if "max_level" in inspect.signature(nx.community.louvain_communities).parameters:
        kwargs["max_level"] = 10
-    communities = nx.community.louvain_communities(G, **kwargs)
+    communities = nx.community.louvain_communities(stable, **kwargs)
    return {node: cid for cid, nodes in enumerate(communities) for node in nodes}


@@ -148,3 +164,54 @@ def cohesion_score(G: nx.Graph, community_nodes: list[str]) -> float:

 def score_all(G: nx.Graph, communities: dict[int, list[str]]) -> dict[int, float]:
    return {cid: cohesion_score(G, nodes) for cid, nodes in communities.items()}
+
+
+def remap_communities_to_previous(
+    communities: dict[int, list[str]],
+    previous_node_community: dict[str, int],
+) -> dict[int, list[str]]:
+    """Remap community IDs to maximize overlap with a previous assignment.
+
+    Uses greedy one-to-one matching by intersection size, then assigns fresh IDs
+    to unmatched communities in deterministic order (size desc, lexical tie-break).
+    """
+    if not communities:
+        return {}
+
+    new_sets = {cid: set(nodes) for cid, nodes in communities.items()}
+    old_sets: dict[int, set[str]] = {}
+    for node, old_cid in previous_node_community.items():
+        old_sets.setdefault(old_cid, set()).add(node)
+
+    overlaps: list[tuple[int, int, int]] = []
+    for old_cid, old_nodes in old_sets.items():
+        for new_cid, new_nodes in new_sets.items():
+            overlap = len(old_nodes & new_nodes)
+            if overlap > 0:
+                overlaps.append((overlap, old_cid, new_cid))
+    overlaps.sort(key=lambda x: (-x[0], x[1], x[2]))
+
+    new_to_final: dict[int, int] = {}
+    used_old_ids: set[int] = set()
+    matched_new_ids: set[int] = set()
+    for _overlap, old_cid, new_cid in overlaps:
+        if old_cid in used_old_ids or new_cid in matched_new_ids:
+            continue
+        new_to_final[new_cid] = old_cid
+        used_old_ids.add(old_cid)
+        matched_new_ids.add(new_cid)
+
+    unmatched = [cid for cid in communities if cid not in matched_new_ids]
+    unmatched.sort(key=lambda cid: (-len(communities[cid]), tuple(sorted(communities[cid]))))
+    next_id = 0
+    for new_cid in unmatched:
+        while next_id in used_old_ids:
+            next_id += 1
+        new_to_final[new_cid] = next_id
+        used_old_ids.add(next_id)
+        next_id += 1
+
+    remapped: dict[int, list[str]] = {}
+    for new_cid, nodes in communities.items():
+        remapped[new_to_final[new_cid]] = sorted(nodes)
+    return dict(sorted(remapped.items(), key=lambda kv: kv[0]))
@@ -3,6 +3,7 @@ from __future__ import annotations
 import contextlib
 import json
 import os
+import re
 import sys
 import time
 from pathlib import Path
@@ -115,16 +116,48 @@ def _relativize_source_files(payload: dict, root: Path) -> None:
                continue


+def _node_community_map(graph_data: dict) -> dict[str, int]:
+    out: dict[str, int] = {}
+    for node in graph_data.get("nodes", []):
+        node_id = node.get("id")
+        cid = node.get("community")
+        if node_id is None or cid is None:
+            continue
+        out[str(node_id)] = int(cid)
+    return out
+
+
+def _canonical_graph_for_compare(graph_data: dict) -> dict:
+    canonical = dict(graph_data)
+    canonical.pop("built_at_commit", None)
+    for key in ("nodes", "links", "edges", "hyperedges"):
+        if key in canonical and isinstance(canonical[key], list):
+            canonical[key] = sorted(
+                canonical[key],
+                key=lambda item: json.dumps(item, sort_keys=True, ensure_ascii=False),
+            )
+    return canonical
+
+
+def _report_for_compare(report_text: str) -> str:
+    return re.sub(r"^- Built from commit: `[^`]+`\n?", "", report_text, flags=re.MULTILINE)
+
+
+def _json_text(data: dict) -> str:
+    return json.dumps(data, indent=2, ensure_ascii=False) + "\n"
+
+
 def _rebuild_code(
    watch_path: Path,
    *,
    changed_paths: list[Path] | None = None,
    follow_symlinks: bool = False,
    force: bool = False,
+    no_cluster: bool = False,
    acquire_lock: bool = True,
    block_on_lock: bool = False,
 ) -> bool:
-    """Re-run AST extraction + build + cluster + report for code files. No LLM needed.
+    """Re-run AST extraction + build + optional cluster + report for code files.

    When ``force`` is True the node-count safety check in ``to_json`` is bypassed
    so the rebuilt graph overwrites graph.json even if it has fewer nodes.
@@ -142,6 +175,9 @@ def _rebuild_code(
    ``block_on_lock=True`` to wait instead of skip (used by the interactive
    ``graphify update`` CLI).

+    ``no_cluster`` skips community detection and writes raw merged extraction
+    JSON to graphify-out/graph.json (mirrors ``extract --no-cluster``).
+
    Returns True on success, False on error or skipped-due-to-lock.
    """
    out = watch_path / _GRAPHIFY_OUT
@@ -156,6 +192,7 @@ def _rebuild_code(
                changed_paths=changed_paths,
                follow_symlinks=follow_symlinks,
                force=force,
+                no_cluster=no_cluster,
                acquire_lock=False,
            )

@@ -166,7 +203,7 @@ def _rebuild_code(
        from graphify.extract import extract, _get_extractor
        from graphify.detect import detect
        from graphify.build import build_from_json
-        from graphify.cluster import cluster, score_all
+        from graphify.cluster import cluster, remap_communities_to_previous, score_all
        from graphify.analyze import god_nodes, surprising_connections, suggest_questions
        from graphify.report import generate
        from graphify.export import to_json, to_html
@@ -225,9 +262,11 @@ def _rebuild_code(
        # source_file matches a path that was changed (re-extracted) or deleted —
        # otherwise the old nodes for those files would survive forever.
        existing_graph = out / "graph.json"
+        existing_graph_data: dict = {}
        if existing_graph.exists():
            try:
                existing = json.loads(existing_graph.read_text(encoding="utf-8"))
+                existing_graph_data = existing
                new_ast_ids = {n["id"] for n in result["nodes"]}
                evict_sources: set[str] = set(deleted_paths)
                if changed_paths is not None:
@@ -257,6 +296,58 @@ def _rebuild_code(
                pass  # corrupt graph.json - proceed with AST-only

        _relativize_source_files(result, project_root)
+        out.mkdir(exist_ok=True)
+        (out / ".graphify_root").write_text(str(watch_root), encoding="utf-8")
+
+        if no_cluster:
+            candidate_graph_data = dict(result)
+            candidate_graph_text = _json_text(candidate_graph_data)
+            existing_text = existing_graph.read_text(encoding="utf-8") if existing_graph.exists() else ""
+            same_graph = False
+            if existing_graph.exists():
+                try:
+                    existing_payload = json.loads(existing_text)
+                    same_graph = (
+                        json.dumps(_canonical_graph_for_compare(existing_payload), sort_keys=True, ensure_ascii=False)
+                        == json.dumps(_canonical_graph_for_compare(candidate_graph_data), sort_keys=True, ensure_ascii=False)
+                    )
+                except Exception:
+                    same_graph = False
+            if not same_graph:
+                if (not force) and existing_graph_data:
+                    existing_n = len(existing_graph_data.get("nodes", []))
+                    new_n = len(candidate_graph_data.get("nodes", []))
+                    if new_n < existing_n:
+                        print(
+                            f"[graphify] WARNING: new graph has {new_n} nodes but existing "
+                            f"graph.json has {existing_n}. Refusing to overwrite — you may be "
+                            f"missing chunk files from a previous session. "
+                            f"Pass force=True to override.",
+                            file=sys.stderr,
+                        )
+                        return False
+                existing_graph.write_text(candidate_graph_text, encoding="utf-8")
+
+            try:
+                from graphify.detect import save_manifest
+                save_manifest(detected["files"])
+            except Exception:
+                pass
+
+            # clear stale needs_update flag if present
+            flag = out / "needs_update"
+            if flag.exists():
+                flag.unlink()
+
+            if same_graph:
+                print("[graphify watch] No code-graph changes detected (--no-cluster); outputs left untouched.")
+            else:
+                print(
+                    "[graphify watch] Rebuilt (no clustering): "
+                    f"{len(result.get('nodes', []))} nodes, {len(result.get('edges', []))} edges"
+                )
+                print(f"[graphify watch] graph.json updated in {out}")
+            return True

        detection = {
            "files": {"code": [str(f) for f in code_files], "document": [], "paper": [], "image": []},
@@ -266,6 +357,9 @@ def _rebuild_code(

        G = build_from_json(result)
        communities = cluster(G)
+        previous_node_community = _node_community_map(existing_graph_data)
+        if previous_node_community:
+            communities = remap_communities_to_previous(communities, previous_node_community)
        cohesion = score_all(G, communities)
        gods = god_nodes(G)
        surprises = surprising_connections(G, communities)
@@ -280,13 +374,52 @@ def _rebuild_code(
            if cid not in labels:
                labels[cid] = "Community " + str(cid)
        questions = suggest_questions(G, communities, labels)
-
-        out.mkdir(exist_ok=True)
-        (out / ".graphify_root").write_text(str(watch_root), encoding="utf-8")
-
-        json_written = to_json(G, communities, str(out / "graph.json"), force=force, built_at_commit=commit)
+        report = generate(G, communities, cohesion, labels, gods, surprises, detection,
+                          {"input": 0, "output": 0}, report_root, suggested_questions=questions,
+                          built_at_commit=commit)
+        report_path = out / "GRAPH_REPORT.md"
+        labels_json = json.dumps({str(k): v for k, v in sorted(labels.items())}, ensure_ascii=False, indent=2) + "\n"
+        graph_tmp = out / ".graph.tmp.json"
+        json_written = to_json(G, communities, str(graph_tmp), force=True, built_at_commit=commit)
        if not json_written:
            return False
+        candidate_graph_data = json.loads(graph_tmp.read_text(encoding="utf-8"))
+        same_graph = False
+        same_report = False
+        if existing_graph.exists():
+            try:
+                existing_payload = json.loads(existing_graph.read_text(encoding="utf-8"))
+                same_graph = (
+                    json.dumps(_canonical_graph_for_compare(existing_payload), sort_keys=True, ensure_ascii=False)
+                    == json.dumps(_canonical_graph_for_compare(candidate_graph_data), sort_keys=True, ensure_ascii=False)
+                )
+            except Exception:
+                same_graph = False
+        if report_path.exists():
+            old_report = report_path.read_text(encoding="utf-8")
+            same_report = _report_for_compare(old_report) == _report_for_compare(report)
+        no_change = same_graph and same_report
+        if no_change:
+            graph_tmp.unlink(missing_ok=True)
+            print("[graphify watch] No code-graph changes detected; graph.json/GRAPH_REPORT.md left untouched.")
+        else:
+            if (not force) and existing_graph_data:
+                existing_n = len(existing_graph_data.get("nodes", []))
+                new_n = len(candidate_graph_data.get("nodes", []))
+                if new_n < existing_n:
+                    graph_tmp.unlink(missing_ok=True)
+                    print(
+                        f"[graphify] WARNING: new graph has {new_n} nodes but existing "
+                        f"graph.json has {existing_n}. Refusing to overwrite — you may be "
+                        f"missing chunk files from a previous session. "
+                        f"Pass force=True to override.",
+                        file=sys.stderr,
+                    )
+                    return False
+            graph_tmp.replace(existing_graph)
+            report_path.write_text(report, encoding="utf-8")
+
+        labels_file.write_text(labels_json, encoding="utf-8")

        try:
            from graphify.detect import save_manifest
@@ -294,27 +427,23 @@ def _rebuild_code(
        except Exception:
            pass

-        report = generate(G, communities, cohesion, labels, gods, surprises, detection,
-                          {"input": 0, "output": 0}, report_root, suggested_questions=questions,
-                          built_at_commit=commit)
-        (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8")
-
        # to_html raises ValueError for graphs > MAX_NODES_FOR_VIZ (5000).
        # Wrap so core outputs (graph.json + GRAPH_REPORT.md) always land.
        html_written = False
-        try:
-            to_html(G, communities, str(out / "graph.html"), community_labels=labels or None)
-            html_written = True
-        except ValueError as viz_err:
-            print(f"[graphify watch] Skipped graph.html: {viz_err}")
-            stale = out / "graph.html"
-            if stale.exists():
-                stale.unlink()
+        if not no_change:
+            try:
+                to_html(G, communities, str(out / "graph.html"), community_labels=labels or None)
+                html_written = True
+            except ValueError as viz_err:
+                print(f"[graphify watch] Skipped graph.html: {viz_err}")
+                stale = out / "graph.html"
+                if stale.exists():
+                    stale.unlink()

        # Regenerate callflow HTML if the user previously generated one —
        # opt-in by existence so users who never ran callflow-html aren't affected.
        callflow_files = list(out.glob("*-callflow.html"))
-        if callflow_files:
+        if callflow_files and not no_change:
            try:
                from graphify.callflow_html import write_callflow_html
                for cf in callflow_files:
@@ -333,12 +462,13 @@ def _rebuild_code(
        if flag.exists():
            flag.unlink()

-        print(f"[graphify watch] Rebuilt: {G.number_of_nodes()} nodes, "
-              f"{G.number_of_edges()} edges, {len(communities)} communities")
-        products = "graph.json" + (", graph.html" if html_written else "") + " and GRAPH_REPORT.md"
-        if callflow_files:
-            products += f", {len(callflow_files)} callflow HTML"
-        print(f"[graphify watch] {products} updated in {out}")
+        if not no_change:
+            print(f"[graphify watch] Rebuilt: {G.number_of_nodes()} nodes, "
+                  f"{G.number_of_edges()} edges, {len(communities)} communities")
+            products = "graph.json" + (", graph.html" if html_written else "") + " and GRAPH_REPORT.md"
+            if callflow_files:
+                products += f", {len(callflow_files)} callflow HTML"
+            print(f"[graphify watch] {products} updated in {out}")
        return True

    except Exception as exc:
@@ -236,3 +236,17 @@ def test_explain_uses_graphify_out_env(tmp_path):
 def test_export_unknown_format_fails(tmp_path):
    r = _run(["export", "pdf"], tmp_path)
    assert r.returncode != 0
+
+
+def test_update_no_cluster_writes_raw_graph(tmp_path):
+    src = tmp_path / "sample.py"
+    src.write_text("def f():\n    return 1\n", encoding="utf-8")
+
+    r = _run(["update", ".", "--no-cluster"], tmp_path)
+    assert r.returncode == 0, r.stderr
+
+    graph_path = tmp_path / "graphify-out" / "graph.json"
+    assert graph_path.exists()
+    data = json.loads(graph_path.read_text(encoding="utf-8"))
+    assert "nodes" in data and "edges" in data
+    assert all("community" not in node for node in data["nodes"])
@@ -3,7 +3,7 @@ import sys
 import networkx as nx
 from pathlib import Path
 from graphify.build import build_from_json
-from graphify.cluster import cluster, cohesion_score, score_all
+from graphify.cluster import cluster, cohesion_score, remap_communities_to_previous, score_all

 FIXTURES = Path(__file__).parent / "fixtures"

@@ -74,3 +74,27 @@ def test_cluster_does_not_write_to_stderr(capsys):
    # Allow logging output (starts with [graphify]) but no raw ANSI codes
    for line in captured.err.splitlines():
        assert "\x1b" not in line, f"cluster() wrote ANSI to stderr: {line!r}"
+
+
+def test_remap_communities_to_previous_reuses_old_ids():
+    communities = {
+        10: ["a", "b", "c"],
+        11: ["d", "e"],
+    }
+    previous = {"a": 5, "b": 5, "c": 5, "d": 1, "e": 1}
+    remapped = remap_communities_to_previous(communities, previous)
+    assert set(remapped.keys()) == {1, 5}
+    assert remapped[5] == ["a", "b", "c"]
+    assert remapped[1] == ["d", "e"]
+
+
+def test_remap_communities_to_previous_assigns_deterministic_new_ids():
+    communities = {
+        7: ["x", "y", "z"],
+        8: ["m"],
+    }
+    previous = {"a": 3}
+    remapped = remap_communities_to_previous(communities, previous)
+    assert list(remapped.keys()) == [0, 1]
+    assert remapped[0] == ["x", "y", "z"]
+    assert remapped[1] == ["m"]
@@ -94,3 +94,36 @@ def test_watch_raises_without_watchdog(tmp_path, monkeypatch):
    from graphify.watch import watch
    with pytest.raises(ImportError, match="watchdog not installed"):
        watch(tmp_path)
+
+
+def test_rebuild_code_is_idempotent_when_cluster_ids_flap(tmp_path, monkeypatch):
+    from graphify import cluster as cluster_mod
+    from graphify.watch import _rebuild_code
+
+    src = tmp_path / "app.py"
+    src.write_text("def alpha():\n    return 1\n\ndef beta():\n    return alpha()\n", encoding="utf-8")
+
+    calls = {"n": 0}
+
+    def flaky_cluster(G):
+        calls["n"] += 1
+        nodes = sorted(G.nodes())
+        if calls["n"] % 2 == 1:
+            return {100: nodes}
+        return {7: nodes}
+
+    monkeypatch.setattr(cluster_mod, "cluster", flaky_cluster)
+    monkeypatch.setattr(cluster_mod, "score_all", lambda _G, comm: {cid: 1.0 for cid in comm})
+
+    assert _rebuild_code(tmp_path)
+    graph_path = tmp_path / "graphify-out" / "graph.json"
+    report_path = tmp_path / "graphify-out" / "GRAPH_REPORT.md"
+    first_graph = graph_path.read_text(encoding="utf-8")
+    first_report = report_path.read_text(encoding="utf-8")
+
+    assert _rebuild_code(tmp_path)
+    second_graph = graph_path.read_text(encoding="utf-8")
+    second_report = report_path.read_text(encoding="utf-8")
+
+    assert first_graph == second_graph
+    assert first_report == second_report