From f117aaccc6a1446b8dff749a2f415e91ed136b7d Mon Sep 17 00:00:00 2001 From: geektan123 <101357603+geektan123@users.noreply.github.com> Date: Wed, 17 Jun 2026 15:16:12 +0530 Subject: [PATCH] feat: PowerShell .psd1 manifest parsing & Import-Module / dot-source edge emission (#1331) (#1341) Index PowerShell .psd1 manifests + emit Import-Module/dot-source edges (closes #1331). Builds on the shipped .psm1 support. Validated: full suite 2107 passed, 18 new tests. Thanks @geektan123. --- graphify/detect.py | 2 +- graphify/extract.py | 239 ++++++++++++++++++++++++++++++- tests/fixtures/sample.psd1 | 13 ++ tests/fixtures/sample_import.ps1 | 10 ++ tests/test_detect.py | 4 + tests/test_languages.py | 141 ++++++++++++++++++ 6 files changed, 407 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/sample.psd1 create mode 100644 tests/fixtures/sample_import.ps1 diff --git a/graphify/detect.py b/graphify/detect.py index 18e58d19..78330c4b 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -26,7 +26,7 @@ class FileType(str, Enum): _MANIFEST_PATH = "graphify-out/manifest.json" -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.razor', '.cshtml', '.cls', '.trigger'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.razor', '.cshtml', '.cls', '.trigger'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index 49dabdcf..fb7741d6 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -7001,6 +7001,8 @@ def extract_powershell(path: Path) -> dict: "using", "return", "if", "else", "elseif", "foreach", "for", "while", "do", "switch", "try", "catch", "finally", "throw", "break", "continue", "exit", "param", "begin", "process", "end", + # Import commands — handled as import edges, not function calls + "import-module", }) def _find_script_block_body(node): @@ -7050,6 +7052,10 @@ def extract_powershell(path: Path) -> dict: body = _find_script_block_body(node) if body: function_bodies.append((func_nid, body)) + # Also walk the body during the main pass so that + # Import-Module / dot-source inside functions emit + # file-level imports_from edges (#1331). + walk(body, parent_class_nid) return if t == "class_statement": @@ -7120,6 +7126,31 @@ def extract_powershell(path: Path) -> dict: return if t == "command": + # Dot-sourcing: `. ./Shared.psm1` + # Uses command_invokation_operator '.' + command_name_expr (not command_name) + invoke_op = next( + (c for c in node.children if c.type == "command_invokation_operator"), None + ) + if invoke_op is not None and _read_text(invoke_op, source).strip() == ".": + name_expr = next( + (c for c in node.children if c.type == "command_name_expr"), None + ) + if name_expr is not None: + name_node = next( + (c for c in name_expr.children if c.type == "command_name"), None + ) + if name_node: + raw_path = _read_text(name_node, source) + # Strip relative path prefix (./ or .\ or just the dot) + module_stem = re.sub(r'^[./\\]+', '', raw_path) + # Drop extension to get bare module name + module_stem = re.sub(r'\.[^.]+$', '', module_stem).replace('\\', '/') + module_name = module_stem.split('/')[-1] + if module_name: + add_edge(file_nid, _make_id(module_name), "imports_from", + node.start_point[0] + 1) + return + cmd_name_node = next((c for c in node.children if c.type == "command_name"), None) if cmd_name_node: cmd_text = _read_text(cmd_name_node, source).lower() @@ -7136,6 +7167,29 @@ def extract_powershell(path: Path) -> dict: module_name = module_tokens[-1].split(".")[-1] add_edge(file_nid, _make_id(module_name), "imports_from", node.start_point[0] + 1) + elif cmd_text == "import-module": + # Collect generic_token args; skip command_parameter flags like -Name + # The module name is the first generic_token (or the one after -Name) + module_name: str | None = None + expect_name = False + for child in node.children: + if child.type != "command_elements": + continue + for el in child.children: + if el.type == "command_parameter": + param_text = _read_text(el, source).lstrip("-").lower() + expect_name = param_text in ("name", "n") + elif el.type == "generic_token": + token = _read_text(el, source) + if module_name is None or expect_name: + module_name = token + expect_name = False + if module_name: + # Strip extension; keep only the stem for the node ID + bare = re.sub(r'\.[^.]+$', '', module_name).split('/')[-1].split('\\')[-1] + if bare: + add_edge(file_nid, _make_id(bare), "imports_from", + node.start_point[0] + 1) return for child in node.children: @@ -7178,10 +7232,192 @@ def extract_powershell(path: Path) -> dict: walk_calls(body_node, caller_nid) clean_edges = [e for e in edges if e["source"] in seen_ids and - (e["target"] in seen_ids or e["relation"] == "imports_from")] + (e["target"] in seen_ids or e["relation"] in ("imports_from", "imports"))] return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls} +# ── PowerShell manifest (.psd1) ────────────────────────────────────────────── + +# Keys in a .psd1 whose values are module names/paths we treat as imports. +_PSD1_IMPORT_KEYS = frozenset({"RootModule", "NestedModules", "RequiredModules"}) + + +def _psd1_collect_string_literals(node, source: bytes) -> list[str]: + """Recursively collect all string_literal text values under *node*.""" + results: list[str] = [] + + def _walk(n) -> None: + if n.type == "string_literal": + raw = source[n.start_byte:n.end_byte].decode(errors="replace") + # Strip surrounding quote chars (' or ") + results.append(raw.strip("'\"")) + return + for child in n.children: + _walk(child) + + _walk(node) + return results + + +def _psd1_module_name(raw: str) -> str: + """Derive a bare module name from a raw string value. + + e.g. 'MyModule.psm1' → 'MyModule', './sub/Util.psm1' → 'Util', 'PSReadLine' → 'PSReadLine' + """ + # Strip path prefix and extension + name = raw.replace("\\", "/").split("/")[-1] + name = re.sub(r"\.[^.]+$", "", name) # remove last extension + return name.strip() + + +def extract_powershell_manifest(path: Path) -> dict: + """Extract module dependency edges from a PowerShell .psd1 manifest file. + + .psd1 files are PowerShell data hashtables, not scripts. tree-sitter-powershell + parses them correctly (they are syntactically valid PS). We walk the AST looking + for RootModule, NestedModules, and RequiredModules keys and emit imports_from + edges for every referenced module. + + RequiredModules supports two forms: + - Simple string: 'PSReadLine' + - Module specification: @{ ModuleName = 'Pester'; ModuleVersion = '5.0' } + For the hashtable form we only follow the ModuleName key. + """ + try: + import tree_sitter_powershell as tsps + from tree_sitter import Language, Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree_sitter_powershell not installed"} + + try: + language = Language(tsps.language()) + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + str_path = str(path) + nodes: list[dict] = [] + edges: list[dict] = [] + seen_ids: set[str] = set() + + def add_node(nid: str, label: str, line: int) -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "code", + "source_file": str_path, "source_location": f"L{line}"}) + + def add_import_edge(src: str, module_raw: str, line: int) -> None: + name = _psd1_module_name(module_raw) + if not name: + return + tgt_nid = _make_id(name) + edges.append({ + "source": src, + "target": tgt_nid, + "relation": "imports_from", + "confidence": "EXTRACTED", + "source_file": str_path, + "source_location": f"L{line}", + "weight": 1.0, + "context": "import", + }) + + file_nid = _make_id(str(path)) + add_node(file_nid, path.name, 1) + + def walk_manifest(node) -> None: + """Walk the AST and emit edges for import-relevant hash_entry nodes.""" + if node.type != "hash_entry": + for child in node.children: + walk_manifest(child) + return + + # Identify the key + key_node = next((c for c in node.children if c.type == "key_expression"), None) + if key_node is None: + return + key_text = source[key_node.start_byte:key_node.end_byte].decode(errors="replace").strip() + + if key_text not in _PSD1_IMPORT_KEYS: + # Still recurse in case there are nested hashes (e.g. ModuleVersion entries + # contain sub-hashes, but we only care about top-level keys for imports) + return + + line = node.start_point[0] + 1 + value_node = next((c for c in node.children if c.type == "pipeline"), None) + if value_node is None: + return + + if key_text == "RootModule": + # Value is a single string + strings = _psd1_collect_string_literals(value_node, source) + for s in strings: + add_import_edge(file_nid, s, line) + + elif key_text == "NestedModules": + # Value is a string or @('a', 'b', ...) array — collect all string literals + strings = _psd1_collect_string_literals(value_node, source) + for s in strings: + add_import_edge(file_nid, s, line) + + elif key_text == "RequiredModules": + # Two forms: + # 1) 'SimpleModule' — direct string literals in the array + # 2) @{ ModuleName = 'Foo'; ModuleVersion = '2.0' } — use ModuleName only + # + # Strategy: walk the value for hash_entry nodes whose key is 'ModuleName'; + # collect their string values. For the remaining string_literal nodes that + # are NOT inside a hash_entry subtree, treat them as simple module names. + module_name_strings: list[str] = [] + inside_hash_entries: set[int] = set() # byte offsets of handled strings + + def find_modulename_entries(n) -> None: + if n.type == "hash_entry": + sub_key = next((c for c in n.children if c.type == "key_expression"), None) + if sub_key is not None: + sk_text = source[sub_key.start_byte:sub_key.end_byte].decode(errors="replace").strip() + # Collect strings inside *all* sub-keys so we can exclude them + for c in n.children: + if c.type == "pipeline": + for s_node in _collect_string_nodes(c): + inside_hash_entries.add(s_node.start_byte) + if sk_text == "ModuleName": + for c in n.children: + if c.type == "pipeline": + for s in _psd1_collect_string_literals(c, source): + module_name_strings.append(s) + return # don't recurse further into this hash_entry + for child in n.children: + find_modulename_entries(child) + + def _collect_string_nodes(n): + """Return all string_literal nodes in subtree.""" + if n.type == "string_literal": + yield n + return + for child in n.children: + yield from _collect_string_nodes(child) + + find_modulename_entries(value_node) + + # Now gather direct string literals not inside hash entries + direct_strings: list[str] = [] + for s_node in _collect_string_nodes(value_node): + if s_node.start_byte not in inside_hash_entries: + raw = source[s_node.start_byte:s_node.end_byte].decode(errors="replace") + direct_strings.append(raw.strip("'\"")) + + for s in direct_strings + module_name_strings: + add_import_edge(file_nid, s, line) + + walk_manifest(root) + + return {"nodes": nodes, "edges": edges, "raw_calls": []} + + # ── Cross-file import resolution ────────────────────────────────────────────── def _source_key(source_file: str, root: Path) -> str: @@ -11773,6 +12009,7 @@ _DISPATCH: dict[str, Any] = { ".zig": extract_zig, ".ps1": extract_powershell, ".psm1": extract_powershell, + ".psd1": extract_powershell_manifest, ".ex": extract_elixir, ".exs": extract_elixir, ".m": extract_objc, diff --git a/tests/fixtures/sample.psd1 b/tests/fixtures/sample.psd1 new file mode 100644 index 00000000..21a90be3 --- /dev/null +++ b/tests/fixtures/sample.psd1 @@ -0,0 +1,13 @@ +@{ + RootModule = 'MyModule.psm1' + ModuleVersion = '1.0.0' + GUID = 'aaaabbbb-cccc-dddd-eeee-ffffffffffff' + Author = 'Test Author' + Description = 'A sample module manifest for graphify tests.' + NestedModules = @('Helpers.psm1', 'Logger.psm1') + RequiredModules = @( + 'PSReadLine', + @{ ModuleName = 'Pester'; ModuleVersion = '5.0' } + ) + FunctionsToExport = @('Get-Data', 'Process-Items') +} diff --git a/tests/fixtures/sample_import.ps1 b/tests/fixtures/sample_import.ps1 new file mode 100644 index 00000000..944233ae --- /dev/null +++ b/tests/fixtures/sample_import.ps1 @@ -0,0 +1,10 @@ +Import-Module Foo +Import-Module -Name Bar.psm1 +. ./Shared.psm1 +. .\Utils.ps1 + +function Invoke-Main { + Import-Module InnerMod + . ./InnerShared.psm1 + Get-Data +} diff --git a/tests/test_detect.py b/tests/test_detect.py index 92297c07..fb8b18d0 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -15,6 +15,10 @@ def test_classify_powershell_module(): # #1315: .psm1 modules were never indexed (CODE_EXTENSIONS gap). assert classify_file(Path("Utils.psm1")) == FileType.CODE +def test_classify_powershell_manifest(): + # #1331: .psd1 manifests must be classified as CODE so the manifest extractor runs. + assert classify_file(Path("MyModule.psd1")) == FileType.CODE + def test_classify_markdown(): assert classify_file(Path("README.md")) == FileType.DOCUMENT diff --git a/tests/test_languages.py b/tests/test_languages.py index ff91d833..11b2b5cb 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -9,6 +9,7 @@ from graphify.extract import ( extract_groovy, extract_sln, extract_csproj, extract_razor, extract_dm, extract_dmi, extract_dmm, extract_dmf, extract_powershell, extract_apex, extract_verilog, + extract_powershell_manifest, ) FIXTURES = Path(__file__).parent / "fixtures" @@ -1063,6 +1064,146 @@ def test_powershell_method_parameter_and_return_type_contexts(): assert ("Save", "void") in _edge_labels(r, "references", "return_type") +# ── PowerShell: Import-Module + dot-source (#1331) ─────────────────────────── + +def test_powershell_import_module_emits_edge(): + """Import-Module Foo at top level emits an imports_from edge.""" + r = extract_powershell(FIXTURES / "sample_import.ps1") + assert "error" not in r + targets = {e["target"] for e in r["edges"] if e["relation"] == "imports_from"} + assert any("foo" in t for t in targets), f"Missing Import-Module Foo edge; targets={targets}" + + +def test_powershell_import_module_with_name_param(): + """Import-Module -Name Bar.psm1 resolves to module stem 'bar'.""" + r = extract_powershell(FIXTURES / "sample_import.ps1") + targets = {e["target"] for e in r["edges"] if e["relation"] == "imports_from"} + assert any("bar" in t for t in targets), f"Missing Import-Module -Name Bar edge; targets={targets}" + + +def test_powershell_dot_source_forward_slash_emits_edge(): + """Dot-source `. ./Shared.psm1` emits an imports_from edge.""" + r = extract_powershell(FIXTURES / "sample_import.ps1") + targets = {e["target"] for e in r["edges"] if e["relation"] == "imports_from"} + assert any("shared" in t for t in targets), f"Missing dot-source Shared edge; targets={targets}" + + +def test_powershell_dot_source_backslash_emits_edge(): + """Dot-source `. .\\Utils.ps1` (backslash path) emits an imports_from edge.""" + r = extract_powershell(FIXTURES / "sample_import.ps1") + targets = {e["target"] for e in r["edges"] if e["relation"] == "imports_from"} + assert any("utils" in t for t in targets), f"Missing dot-source Utils edge; targets={targets}" + + +def test_powershell_import_module_inside_function_emits_edge(): + """Import-Module inside a function body still produces an imports_from edge.""" + r = extract_powershell(FIXTURES / "sample_import.ps1") + targets = {e["target"] for e in r["edges"] if e["relation"] == "imports_from"} + assert any("innermod" in t for t in targets), ( + f"Missing Import-Module InnerMod edge from function body; targets={targets}" + ) + + +def test_powershell_import_module_not_a_raw_call(): + """Import-Module must not appear in raw_calls (it is an import, not a function call).""" + r = extract_powershell(FIXTURES / "sample_import.ps1") + import_module_calls = [ + rc for rc in r.get("raw_calls", []) + if rc.get("callee", "").lower() == "import-module" + ] + assert not import_module_calls, ( + f"Import-Module appeared in raw_calls but should be emitted as import edge: {import_module_calls}" + ) + + +def test_powershell_dot_source_inside_function_emits_edge(): + """Dot-source inside a function body still produces an imports_from edge.""" + r = extract_powershell(FIXTURES / "sample_import.ps1") + targets = {e["target"] for e in r["edges"] if e["relation"] == "imports_from"} + assert any("innershared" in t for t in targets), ( + f"Missing dot-source InnerShared edge from function body; targets={targets}" + ) + + +# ── PowerShell manifest (.psd1) (#1331) ────────────────────────────────────── + +def test_powershell_psd1_dispatched(): + """_get_extractor should route .psd1 to extract_powershell_manifest.""" + from graphify.extract import _get_extractor + import tempfile, os + with tempfile.NamedTemporaryFile(suffix=".psd1", delete=False) as f: + f.write(b"@{ RootModule = 'X.psm1' }") + path = f.name + try: + assert _get_extractor(Path(path)) is extract_powershell_manifest + finally: + os.unlink(path) + + +def test_powershell_psd1_no_error(): + r = extract_powershell_manifest(FIXTURES / "sample.psd1") + assert "error" not in r + + +def test_powershell_psd1_has_file_node(): + r = extract_powershell_manifest(FIXTURES / "sample.psd1") + assert any("sample.psd1" in n["label"] for n in r["nodes"]), ( + f"Missing file node for sample.psd1; nodes={[n['label'] for n in r['nodes']]}" + ) + + +def test_powershell_psd1_root_module(): + """RootModule = 'MyModule.psm1' produces an imports_from edge to 'mymodule'.""" + r = extract_powershell_manifest(FIXTURES / "sample.psd1") + targets = {e["target"] for e in r["edges"] if e["relation"] == "imports_from"} + assert any("mymodule" in t for t in targets), ( + f"Missing RootModule edge for MyModule; targets={targets}" + ) + + +def test_powershell_psd1_nested_modules(): + """NestedModules = @('Helpers.psm1', 'Logger.psm1') produces edges for both.""" + r = extract_powershell_manifest(FIXTURES / "sample.psd1") + targets = {e["target"] for e in r["edges"] if e["relation"] == "imports_from"} + assert any("helpers" in t for t in targets), f"Missing NestedModules Helpers edge; targets={targets}" + assert any("logger" in t for t in targets), f"Missing NestedModules Logger edge; targets={targets}" + + +def test_powershell_psd1_required_modules_string(): + """RequiredModules string form 'PSReadLine' produces an imports_from edge.""" + r = extract_powershell_manifest(FIXTURES / "sample.psd1") + targets = {e["target"] for e in r["edges"] if e["relation"] == "imports_from"} + assert any("psreadline" in t for t in targets), ( + f"Missing RequiredModules PSReadLine edge; targets={targets}" + ) + + +def test_powershell_psd1_required_modules_hashtable(): + """RequiredModules hashtable form @{{ ModuleName='Pester' }} produces an imports_from edge.""" + r = extract_powershell_manifest(FIXTURES / "sample.psd1") + targets = {e["target"] for e in r["edges"] if e["relation"] == "imports_from"} + assert any("pester" in t for t in targets), ( + f"Missing RequiredModules Pester (hashtable form) edge; targets={targets}" + ) + + +def test_powershell_psd1_no_moduleversion_as_edge(): + """ModuleVersion values ('5.0', '1.0.0') must NOT appear as import targets.""" + r = extract_powershell_manifest(FIXTURES / "sample.psd1") + targets = {e["target"] for e in r["edges"] if e["relation"] == "imports_from"} + assert not any(t in targets for t in ("5_0", "1_0_0", "5.0", "1.0.0")), ( + f"ModuleVersion string leaked into import targets: {targets}" + ) + + +def test_powershell_psd1_no_dangling_edges(): + """All imports_from edge sources must exist in the node set.""" + r = extract_powershell_manifest(FIXTURES / "sample.psd1") + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + assert e["source"] in node_ids, f"Dangling source in edge: {e}" + + # ── TypeScript dynamic imports ─────────────────────────────────────────────── def test_ts_dynamic_import_no_error():