From adb52a1ea0b65854158e49f9788b91ef4c8d97fc Mon Sep 17 00:00:00 2001 From: Safi Date: Tue, 16 Jun 2026 02:12:53 +0100 Subject: [PATCH] Index .psm1, anchor Swift import targets, dedupe no-cluster edges - #1315: add .psm1 to CODE_EXTENSIONS + _DISPATCH so PowerShell modules are indexed - #1327: synthesize a module node for Swift import targets (new LanguageConfig flag synthesize_import_module_nodes) so imports edges survive build.py pruning; strengthen the Swift dangling-edge test to also assert edge targets - #1317: dedupe parallel edges by (source,target,relation) in the --no-cluster and incremental update write paths so edge counts are deterministic and `update` is idempotent Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/__main__.py | 4 ++++ graphify/build.py | 23 +++++++++++++++++++++++ graphify/detect.py | 2 +- graphify/extract.py | 29 +++++++++++++++++++++++++++++ graphify/watch.py | 9 +++++++-- tests/test_build.py | 28 +++++++++++++++++++++++++++- tests/test_detect.py | 4 ++++ tests/test_languages.py | 36 ++++++++++++++++++++++++++++++++++++ 8 files changed, 131 insertions(+), 4 deletions(-) diff --git a/graphify/__main__.py b/graphify/__main__.py index 60a288559..91c2272d6 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -4425,7 +4425,11 @@ def main() -> None: if no_cluster: # --no-cluster: dump the raw merged extraction as graph.json. # No NetworkX, no community detection, no analysis sidecar. + # Dedupe parallel edges so counts match the clustered path (whose + # DiGraph collapses them) and stay deterministic across modes (#1317). + from graphify.build import dedupe_edges as _dedupe_edges from graphify.export import backup_if_protected as _backup + merged["edges"] = _dedupe_edges(merged["edges"]) _backup(graphify_out) graph_json_path.write_text( json.dumps(merged, indent=2), encoding="utf-8" diff --git a/graphify/build.py b/graphify/build.py index 0c47d419d..a66a55337 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -104,6 +104,29 @@ def edge_datas(G: nx.Graph, u: str, v: str) -> list[dict]: return [raw] +def dedupe_edges(edges: list[dict]) -> list[dict]: + """Collapse exact parallel edges by ``(source, target, relation)``, keeping the + first occurrence. + + The clustered build path runs edges through a NetworkX ``DiGraph``, which + collapses parallel edges automatically. The ``--no-cluster`` and incremental + ``update`` write paths bypass NetworkX and concatenate edge lists raw, so + duplicates accumulate and edge counts become non-deterministic across build + modes / repeated updates (#1317). Deduping on the connectivity identity is + zero-signal-loss and restores idempotency. Callers that intentionally keep + parallel edges (multigraph output) must not use this. + """ + seen: set[tuple] = set() + out: list[dict] = [] + for e in edges: + key = (e.get("source"), e.get("target"), e.get("relation")) + if key in seen: + continue + seen.add(key) + out.append(e) + return out + + def build_from_json(extraction: dict, *, directed: bool = False, root: str | Path | None = None) -> nx.Graph: """Build a NetworkX graph from an extraction dict. diff --git a/graphify/detect.py b/graphify/detect.py index 3f6a59b36..18e58d191 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -26,7 +26,7 @@ class FileType(str, Enum): _MANIFEST_PATH = "graphify-out/manifest.json" -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.razor', '.cshtml', '.cls', '.trigger'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.razor', '.cshtml', '.cls', '.trigger'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index de7a75349..55049ba46 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -466,6 +466,12 @@ class LanguageConfig: # Extra walk hook called after generic dispatch (for JS arrow functions, C# namespaces, etc.) extra_walk_fn: Callable | None = None + # When True, synthesize a node for each `imports` edge target an import_handler + # emits (carrying an `_import_label` key). Languages whose imports name modules + # rather than resolvable files (e.g. Swift `import CoreKit`) need this, else the + # edge is pruned in build.py for pointing at a non-existent target node. + synthesize_import_module_nodes: bool = False + # ── Generic helpers ─────────────────────────────────────────────────────────── @@ -2229,6 +2235,9 @@ def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, st "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, + # Consumed by import-node synthesis at the walk() call site + # (LanguageConfig.synthesize_import_module_nodes); see #1327. + "_import_label": raw, }) break @@ -2267,6 +2276,7 @@ _SWIFT_CONFIG = LanguageConfig( body_fallback_child_types=("class_body", "protocol_body", "function_body", "enum_class_body"), function_boundary_types=frozenset({"function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}), import_handler=_import_swift, + synthesize_import_module_nodes=True, ) # ── Generic extractor ───────────────────────────────────────────────────────── @@ -2372,7 +2382,25 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict: # Import types if t in config.import_types: if config.import_handler: + _imp_before = len(edges) config.import_handler(node, source, file_nid, stem, edges, str_path) + if config.synthesize_import_module_nodes: + # Imports that name a module (not a resolvable file) point at a + # synthetic target node; create it so build.py keeps the edge (#1327). + for _e in edges[_imp_before:]: + _lbl = _e.pop("_import_label", None) + if _lbl is None or _e.get("relation") != "imports": + continue + _tgt = _e["target"] + if _tgt not in seen_ids: + seen_ids.add(_tgt) + nodes.append({ + "id": _tgt, + "label": _lbl, + "file_type": "code", + "source_file": str_path, + "source_location": _e.get("source_location", "L1"), + }) # For export_statement: only return (skip children) if it's a re-export # (has a `from` source). Otherwise fall through to walk children which may # contain function_declaration, class_declaration, etc. @@ -11568,6 +11596,7 @@ _DISPATCH: dict[str, Any] = { ".toc": extract_lua, ".zig": extract_zig, ".ps1": extract_powershell, + ".psm1": extract_powershell, ".ex": extract_elixir, ".exs": extract_elixir, ".m": extract_objc, diff --git a/graphify/watch.py b/graphify/watch.py index b6eec5126..b8810f504 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -584,9 +584,13 @@ def _rebuild_code( if no_cluster: # Normalise to "links" key so schema is consistent with the full clustered path. + # Dedupe parallel edges (the clustered path's DiGraph collapses them implicitly); + # without it, --no-cluster + repeated `update` accumulate duplicates and edge + # counts diverge across build modes (#1317). + from graphify.build import dedupe_edges as _dedupe_edges candidate_graph_data = { **{k: v for k, v in result.items() if k != "edges"}, - "links": result.get("edges", []), + "links": _dedupe_edges(result.get("edges", [])), } candidate_graph_text = _json_text(candidate_graph_data) same_graph = False @@ -624,7 +628,8 @@ def _rebuild_code( else: print( "[graphify watch] Rebuilt (no clustering): " - f"{len(result.get('nodes', []))} nodes, {len(result.get('edges', []))} edges" + f"{len(candidate_graph_data.get('nodes', []))} nodes, " + f"{len(candidate_graph_data.get('links', []))} edges" ) print(f"[graphify watch] graph.json updated in {out}") return True diff --git a/tests/test_build.py b/tests/test_build.py index c57d7a769..6c3b7f31a 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -2,10 +2,36 @@ import json from pathlib import Path import networkx as nx from networkx.readwrite import json_graph -from graphify.build import build_from_json, build, build_merge, edge_data, edge_datas +from graphify.build import build_from_json, build, build_merge, edge_data, edge_datas, dedupe_edges FIXTURES = Path(__file__).parent / "fixtures" + +def test_dedupe_edges_collapses_exact_parallels(): + # #1317: --no-cluster / incremental update concatenate edge lists raw. + edges = [ + {"source": "a", "target": "b", "relation": "calls", "source_location": "L1"}, + {"source": "a", "target": "b", "relation": "calls", "source_location": "L9"}, # dup + {"source": "a", "target": "b", "relation": "imports"}, # different relation: kept + {"source": "b", "target": "c", "relation": "calls"}, + ] + out = dedupe_edges(edges) + keys = [(e["source"], e["target"], e["relation"]) for e in out] + assert keys == [("a", "b", "calls"), ("a", "b", "imports"), ("b", "c", "calls")] + # first occurrence wins (keeps L1, not L9) + assert out[0]["source_location"] == "L1" + + +def test_dedupe_edges_is_idempotent(): + edges = [ + {"source": "a", "target": "b", "relation": "calls"}, + {"source": "a", "target": "b", "relation": "calls"}, + ] + once = dedupe_edges(edges) + twice = dedupe_edges(once + edges) # simulate a second `update` re-concatenating + assert len(once) == 1 + assert len(twice) == 1 + def load_extraction(): return json.loads((FIXTURES / "extraction.json").read_text()) diff --git a/tests/test_detect.py b/tests/test_detect.py index 0c8ef484b..92297c075 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -11,6 +11,10 @@ def test_classify_python(): def test_classify_typescript(): assert classify_file(Path("bar.ts")) == FileType.CODE +def test_classify_powershell_module(): + # #1315: .psm1 modules were never indexed (CODE_EXTENSIONS gap). + assert classify_file(Path("Utils.psm1")) == FileType.CODE + def test_classify_markdown(): assert classify_file(Path("README.md")) == FileType.DOCUMENT diff --git a/tests/test_languages.py b/tests/test_languages.py index 91ab2a170..ee5d4a391 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -596,6 +596,28 @@ def test_swift_no_dangling_edges(): node_ids = {n["id"] for n in r["nodes"]} for e in r["edges"]: assert e["source"] in node_ids + # #1327: targets must resolve to a node too, else build.py prunes the edge. + assert e["target"] in node_ids, f"dangling target {e['target']} ({e['relation']})" + + +def test_swift_imports_survive_build(): + # #1327: `import Foundation` / `import UIKit` previously emitted edges to bare + # module ids with no backing node, so build.py dropped 100% of Swift imports. + from graphify.build import build_from_json + r = extract_swift(FIXTURES / "sample.swift") + import_edges = [e for e in r["edges"] if e["relation"] == "imports"] + assert import_edges, "extractor should emit Swift import edges" + node_ids = {n["id"] for n in r["nodes"]} + for e in import_edges: + assert e["target"] in node_ids # synthesized module node exists + # No private bookkeeping key should leak into output edges. + assert all("_import_label" not in e for e in r["edges"]) + # Edges must survive the build (which prunes edges with unknown endpoints). + G = build_from_json(r) + surviving = [ + (u, v) for u, v, d in G.edges(data=True) if d.get("relation") == "imports" + ] + assert surviving, "Swift import edges must survive build_from_json (#1327)" def test_swift_finds_actor(): r = extract_swift(FIXTURES / "sample.swift") @@ -1005,6 +1027,20 @@ def test_powershell_no_error(): assert "error" not in r +def test_powershell_psm1_dispatched_and_extracted(tmp_path): + # #1315: .psm1 modules were never indexed — no dispatch entry, no CODE_EXTENSIONS. + from graphify.extract import _get_extractor + mod = tmp_path / "Utils.psm1" + mod.write_text( + "function Get-Greeting { param([string]$Name) return \"Hi $Name\" }\n", + encoding="utf-8", + ) + assert _get_extractor(mod) is extract_powershell + r = extract_powershell(mod) + assert "error" not in r + assert any("Get-Greeting" in n["label"] for n in r["nodes"]) + + def test_powershell_finds_class_and_method(): r = extract_powershell(FIXTURES / "sample.ps1") labels = [n["label"] for n in r["nodes"]]