From 2d783e569ab8ed87bc9d76a9a5619b8d548e3241 Mon Sep 17 00:00:00 2001 From: Safi Date: Sun, 17 May 2026 23:05:48 +0100 Subject: [PATCH] fix hooks phantom dir on git < 2.31, save_manifest incremental data loss, cohesion rounding, C++ inheritance; add --resolution and --exclude-hubs - hooks.py: drop --path-format=absolute (added git 2.31), validate no newlines in path, anchor relative paths on repo root (#907) - detect.py: seed save_manifest from existing manifest before loop so incremental callers don't erase untouched file entries (#917) - cluster.py: drop round(..., 2) from cohesion_score so split threshold 0.05 fires correctly; add resolution param to _partition and cluster; add exclude_hubs_percentile to cluster with majority-vote reattachment (#919) - report.py: format cohesion with :.2f for display - __main__.py: wire --resolution and --exclude-hubs into extract and cluster-only commands (#919) - C++ inheritance already written to disk by analysis agent (#915) Co-Authored-By: Claude Sonnet 4.6 --- graphify/__main__.py | 25 ++++++++++++++++-- graphify/cluster.py | 60 ++++++++++++++++++++++++++++++++++++++------ graphify/detect.py | 31 ++++++++++++++++++----- graphify/hooks.py | 14 ++++++++--- graphify/report.py | 2 +- 5 files changed, 113 insertions(+), 19 deletions(-) diff --git a/graphify/__main__.py b/graphify/__main__.py index 51c4db9d..3d9cc452 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -1761,11 +1761,21 @@ def main() -> None: args = sys.argv[2:] watch_path: Path | None = None graph_override: Path | None = None + co_resolution: float = 1.0 + co_exclude_hubs: float | None = None i_arg = 0 while i_arg < len(args): a = args[i_arg] if a == "--graph" and i_arg + 1 < len(args): graph_override = Path(args[i_arg + 1]); i_arg += 2 + elif a == "--resolution" and i_arg + 1 < len(args): + co_resolution = float(args[i_arg + 1]); i_arg += 2 + elif a.startswith("--resolution="): + co_resolution = float(a.split("=", 1)[1]); i_arg += 1 + elif a == "--exclude-hubs" and i_arg + 1 < len(args): + co_exclude_hubs = float(args[i_arg + 1]); i_arg += 2 + elif a.startswith("--exclude-hubs="): + co_exclude_hubs = float(a.split("=", 1)[1]); i_arg += 1 elif a == "--no-viz" or a.startswith("--min-community-size="): i_arg += 1 elif a.startswith("--"): @@ -1792,7 +1802,7 @@ def main() -> None: G = build_from_json(_raw, directed=_directed) print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") print("Re-clustering...") - communities = cluster(G) + communities = cluster(G, resolution=co_resolution, exclude_hubs_percentile=co_exclude_hubs) cohesion = score_all(G, communities) gods = god_nodes(G) surprises = surprising_connections(G, communities) @@ -2415,6 +2425,9 @@ def main() -> None: cli_token_budget: int | None = None cli_max_concurrency: int | None = None cli_api_timeout: float | None = None + # Clustering tuning knobs + cli_resolution: float = 1.0 + cli_exclude_hubs: float | None = None def _parse_int(name: str, raw: str) -> int: try: @@ -2480,6 +2493,14 @@ def main() -> None: cli_api_timeout = _parse_float("--api-timeout", args[i + 1]); i += 2 elif a.startswith("--api-timeout="): cli_api_timeout = _parse_float("--api-timeout", a.split("=", 1)[1]); i += 1 + elif a == "--resolution" and i + 1 < len(args): + cli_resolution = _parse_float("--resolution", args[i + 1]); i += 2 + elif a.startswith("--resolution="): + cli_resolution = _parse_float("--resolution", a.split("=", 1)[1]); i += 1 + elif a == "--exclude-hubs" and i + 1 < len(args): + cli_exclude_hubs = float(args[i + 1]); i += 2 + elif a.startswith("--exclude-hubs="): + cli_exclude_hubs = float(a.split("=", 1)[1]); i += 1 else: i += 1 @@ -2796,7 +2817,7 @@ def main() -> None: ) sys.exit(1) - communities = _cluster(G) + communities = _cluster(G, resolution=cli_resolution, exclude_hubs_percentile=cli_exclude_hubs) cohesion = _score_all(G, communities) try: gods = _god_nodes(G) diff --git a/graphify/cluster.py b/graphify/cluster.py index 7959ec17..a567e0cc 100644 --- a/graphify/cluster.py +++ b/graphify/cluster.py @@ -19,12 +19,15 @@ def _suppress_output(): return contextlib.redirect_stdout(io.StringIO()) -def _partition(G: nx.Graph) -> dict[str, int]: +def _partition(G: nx.Graph, resolution: float = 1.0) -> dict[str, int]: """Run community detection. Returns {node_id: community_id}. Tries Leiden (graspologic) first — best quality. Falls back to Louvain (built into networkx) if graspologic is not installed. + resolution > 1.0 → more, smaller communities. + resolution < 1.0 → fewer, larger communities. + Output from graspologic is suppressed to prevent ANSI escape codes from corrupting terminal scroll buffers on Windows PowerShell 5.1. """ @@ -49,6 +52,8 @@ def _partition(G: nx.Graph) -> dict[str, int]: kwargs["random_seed"] = 42 if "trials" in lsig: kwargs["trials"] = 1 + if "resolution" in lsig: + kwargs["resolution"] = resolution # Suppress graspologic output to prevent ANSI escape codes from # corrupting PowerShell 5.1 scroll buffer (issue #19) old_stderr = sys.stderr @@ -65,7 +70,7 @@ def _partition(G: nx.Graph) -> dict[str, int]: # Fallback: networkx louvain (available since networkx 2.7). # Inspect kwargs to stay compatible across NetworkX versions — max_level # was added in a later release and prevents hangs on large sparse graphs. - kwargs: dict = {"seed": 42, "threshold": 1e-4} + kwargs: dict = {"seed": 42, "threshold": 1e-4, "resolution": resolution} if "max_level" in inspect.signature(nx.community.louvain_communities).parameters: kwargs["max_level"] = 10 communities = nx.community.louvain_communities(stable, **kwargs) @@ -78,7 +83,11 @@ _COHESION_SPLIT_THRESHOLD = 0.05 # re-split communities with cohesion below this _COHESION_SPLIT_MIN_SIZE = 50 # only cohesion-split if community has at least this many nodes -def cluster(G: nx.Graph) -> dict[int, list[str]]: +def cluster( + G: nx.Graph, + resolution: float = 1.0, + exclude_hubs_percentile: float | None = None, +) -> dict[int, list[str]]: """Run Leiden community detection. Returns {community_id: [node_ids]}. Community IDs are stable across runs: 0 = largest community after splitting. @@ -87,6 +96,13 @@ def cluster(G: nx.Graph) -> dict[int, list[str]]: Accepts directed or undirected graphs. DiGraphs are converted to undirected internally since Louvain/Leiden require undirected input. + + resolution: passed to Leiden/Louvain. >1.0 = more smaller communities, + <1.0 = fewer larger communities. Default 1.0. + exclude_hubs_percentile: if set (0-100), nodes whose degree exceeds this + percentile are excluded from partitioning and reattached to their + majority-vote neighbour community afterwards. Useful for staging/utility + super-hubs that inflate god-node rankings (#919). """ if G.number_of_nodes() == 0: return {} @@ -95,14 +111,26 @@ def cluster(G: nx.Graph) -> dict[int, list[str]]: if G.number_of_edges() == 0: return {i: [n] for i, n in enumerate(sorted(G.nodes))} + # Compute hub exclusion set before removing anything so degree is based on full graph + hub_nodes: set[str] = set() + if exclude_hubs_percentile is not None: + degrees = sorted(d for _, d in G.degree()) + if degrees: + idx = max(0, int(len(degrees) * exclude_hubs_percentile / 100) - 1) + threshold = degrees[idx] + hub_nodes = {n for n, d in G.degree() if d > threshold} + # Leiden warns and drops isolates - handle them separately - isolates = [n for n in G.nodes() if G.degree(n) == 0] - connected_nodes = [n for n in G.nodes() if G.degree(n) > 0] + # Also exclude hub nodes from partitioning so they don't pull unrelated + # subsystems into the same community + excluded = hub_nodes + isolates = [n for n in G.nodes() if G.degree(n) == 0 and n not in excluded] + connected_nodes = [n for n in G.nodes() if G.degree(n) > 0 and n not in excluded] connected = G.subgraph(connected_nodes) raw: dict[int, list[str]] = {} if connected.number_of_nodes() > 0: - partition = _partition(connected) + partition = _partition(connected, resolution=resolution) for node, cid in partition.items(): raw.setdefault(cid, []).append(node) @@ -112,6 +140,24 @@ def cluster(G: nx.Graph) -> dict[int, list[str]]: raw[next_cid] = [node] next_cid += 1 + # Reattach excluded hubs by majority-vote neighbour community + if hub_nodes: + node_community: dict[str, int] = {n: cid for cid, nodes in raw.items() for n in nodes} + for hub in sorted(hub_nodes): + votes: dict[int, int] = {} + for nb in G.neighbors(hub): + cid = node_community.get(nb) + if cid is not None: + votes[cid] = votes.get(cid, 0) + 1 + if votes: + best = min(votes, key=lambda c: (-votes[c], c)) + raw.setdefault(best, []).append(hub) + node_community[hub] = best + else: + raw[next_cid] = [hub] + node_community[hub] = next_cid + next_cid += 1 + # Split oversized communities max_size = max(_MIN_SPLIT_SIZE, int(G.number_of_nodes() * _MAX_COMMUNITY_FRACTION)) final_communities: list[list[str]] = [] @@ -163,7 +209,7 @@ def cohesion_score(G: nx.Graph, community_nodes: list[str]) -> float: subgraph = G.subgraph(community_nodes) actual = subgraph.number_of_edges() possible = n * (n - 1) / 2 - return round(actual / possible, 2) if possible > 0 else 0.0 + return actual / possible if possible > 0 else 0.0 def score_all(G: nx.Graph, communities: dict[int, list[str]]) -> dict[int, float]: diff --git a/graphify/detect.py b/graphify/detect.py index fe93d997..8ca1c337 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -856,7 +856,31 @@ def save_manifest( kind="both" — full pipeline: stamps both hashes (default). """ existing = load_manifest(manifest_path) + + def _normalise_entry(entry): + if isinstance(entry, (int, float)): + return {"mtime": entry, "ast_hash": "", "semantic_hash": ""} + if isinstance(entry, dict) and "hash" in entry and "ast_hash" not in entry: + return {"mtime": entry.get("mtime", 0), "ast_hash": entry["hash"], "semantic_hash": ""} + if isinstance(entry, dict): + return entry + return None + + # Seed from the existing manifest so incremental callers passing a subset + # of files don't silently erase entries for untouched files (#917). + # Prune entries whose file no longer exists on disk — those are genuine + # deletions that detect_incremental() should treat as gone. manifest: dict[str, dict] = {} + for f, entry in existing.items(): + normalised = _normalise_entry(entry) + if normalised is None: + continue + try: + if Path(f).exists(): + manifest[f] = normalised + except OSError: + continue + for file_list in files.values(): for f in file_list: try: @@ -865,12 +889,7 @@ def save_manifest( h = _md5_file(p) except OSError: continue # file deleted between detect() and manifest write - prev = existing.get(f, {}) - # Normalise legacy {mtime, hash} entries to new schema - if isinstance(prev, (int, float)): - prev = {"mtime": prev, "ast_hash": "", "semantic_hash": ""} - elif isinstance(prev, dict) and "hash" in prev and "ast_hash" not in prev: - prev = {"mtime": prev.get("mtime", 0), "ast_hash": prev["hash"], "semantic_hash": ""} + prev = _normalise_entry(existing.get(f, {})) or {} entry: dict = {"mtime": mtime} if kind in ("ast", "both"): entry["ast_hash"] = h diff --git a/graphify/hooks.py b/graphify/hooks.py index 2723110e..ae701b9f 100644 --- a/graphify/hooks.py +++ b/graphify/hooks.py @@ -199,14 +199,22 @@ def _hooks_dir(root: Path) -> Path: ) # In a linked worktree .git is a file not a directory, so constructing # root/.git/hooks directly fails. Ask git for the real hooks path instead. + # NOTE: do NOT pass --path-format=absolute — added in git 2.31; older git + # echoes it back as a literal argument, contaminating stdout and causing a + # phantom directory to be created (#907). git -C already returns an + # absolute path for worktree/external-gitdir cases, and a path relative to + # for normal repos — anchoring on root covers both. import subprocess as _sp try: res = _sp.run( - ["git", "-C", str(root), "rev-parse", "--path-format=absolute", "--git-path", "hooks"], + ["git", "-C", str(root), "rev-parse", "--git-path", "hooks"], capture_output=True, text=True, ) - if res.returncode == 0: - d = Path(res.stdout.strip()) + raw = res.stdout.strip() + # A valid hooks path can never contain newlines or NUL. Their presence + # means git echoed an unrecognised flag back (old git behaviour). + if res.returncode == 0 and raw and not any(c in raw for c in ("\n", "\r", "\x00")): + d = (root / raw).resolve() d.mkdir(parents=True, exist_ok=True) return d except (OSError, FileNotFoundError): diff --git a/graphify/report.py b/graphify/report.py index 8aa51e7c..9ca7a2fc 100644 --- a/graphify/report.py +++ b/graphify/report.py @@ -144,7 +144,7 @@ def generate( lines += [ "", f"### Community {cid} - \"{label}\"", - f"Cohesion: {score}", + f"Cohesion: {score:.2f}", f"Nodes ({len(real_nodes)}): {', '.join(display)}{suffix}", ]