diff --git a/graphify/extract.py b/graphify/extract.py index ceb80ed8..6054fca3 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4541,7 +4541,7 @@ def extract_dart(path: Path) -> dict: mixin_clean = mixin.split("<")[0].strip() mixin_nid = _make_id(mixin_clean) add_node(mixin_nid, mixin_clean, source_file=None) - add_edge(class_nid, mixin_nid, "implements") + add_edge(class_nid, mixin_nid, "mixes_in") # Map interfaces for interface in interfaces_list: @@ -4823,8 +4823,209 @@ def extract_dart(path: Path) -> dict: return {"nodes": nodes, "edges": edges} +def _sv_first_identifier(node, source: bytes) -> str | None: + """First `simple_identifier` under node in pre-order, or None. + + tree-sitter-verilog 1.0.3 nests declaration names a few levels deep instead + of exposing a `name` field. Scope the search to the right child node (e.g. + `function_identifier`) or this returns the return-type instead of the name. + """ + if node is None: + return None + for child in node.children: + if child.type == "simple_identifier": + return _read_text(child, source) + found = _sv_first_identifier(child, source) + if found: + return found + return None + + +def _sv_child(node, type_name: str) -> object | None: + if node is None: + return None + for child in node.children: + if child.type == type_name: + return child + return None + + +_SV_BUILTIN_TYPES = frozenset({ + "bit", "logic", "reg", "wire", "int", "integer", "shortint", "longint", + "byte", "time", "real", "shortreal", "void", "string", "type", "event", + "mailbox", "semaphore", "process", "chandle", +}) + +_SV_NON_TYPE_WORDS = frozenset({ + "return", "if", "else", "for", "foreach", "while", "case", "begin", "end", + "function", "task", "class", "endclass", "endfunction", "endtask", +}) + +# One level of balanced parens (e.g. `Foo #(Bar #(int))`) — bounded so malformed +# input cannot trigger pathological backtracking. +_SV_PARENS_INNER = r"(?:[^()]|\([^()]*\))*" +_SV_PARENS = r"\(" + _SV_PARENS_INNER + r"\)" + +_SV_FUNC_RE = re.compile( + r"\bfunction\s+([A-Za-z_]\w*(?:\s*#\s*" + _SV_PARENS + r")?)\s+(\w+)\s*" + r"\((" + _SV_PARENS_INNER + r")\)\s*;", + re.MULTILINE, +) + +_SV_PARAM_RE = re.compile( + r"\s*(?:input|output|inout|ref|const\s+ref)?\s*" + r"([A-Za-z_]\w*(?:\s*#\s*" + _SV_PARENS + r")?)\s+\w+" +) + + +def _sv_strip_comments(text: str) -> str: + text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL) + return re.sub(r"//.*", "", text) + + +def _sv_split_type_list(text: str) -> list[str]: + parts: list[str] = [] + depth = 0 + start = 0 + for idx, ch in enumerate(text): + if ch == "(": + depth += 1 + elif ch == ")": + depth = max(0, depth - 1) + elif ch == "," and depth == 0: + item = text[start:idx].strip() + if item: + parts.append(item) + start = idx + 1 + item = text[start:].strip() + if item: + parts.append(item) + return parts + + +def _sv_collect_type_refs(type_text: str, generic: bool = False, + skip: frozenset[str] = frozenset()) -> list[tuple[str, str]]: + refs: list[tuple[str, str]] = [] + text = type_text.strip() + if not text: + return refs + head = re.match(r"([A-Za-z_]\w*)", text) + if head: + name = head.group(1) + # `skip` carries the enclosing class's `#(type T = ...)` parameters so + # they are not mistaken for referenced types. + if name not in _SV_BUILTIN_TYPES and name not in _SV_NON_TYPE_WORDS and name not in skip: + refs.append((name, "generic_arg" if generic else "type")) + params = re.search(r"#\s*\((" + _SV_PARENS_INNER + r")\)", text) + if params: + for arg in _sv_split_type_list(params.group(1)): + refs.extend(_sv_collect_type_refs(arg, generic=True, skip=skip)) + return refs + + +def _augment_systemverilog_semantics( + raw: str, + stem: str, + str_path: str, + file_nid: str, + nodes: list[dict], + edges: list[dict], + seen_ids: set[str], +) -> None: + label_to_nid = {node["label"]: node["id"] for node in nodes} + + def line_for(offset: int) -> int: + return raw.count("\n", 0, offset) + 1 + + def add_node(nid: str, label: str, line: int) -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "code", + "source_file": str_path, "source_location": f"L{line}", + "confidence_score": 1.0}) + label_to_nid[label] = nid + + def ensure_type(label: str, line: int) -> str: + if label in label_to_nid: + return label_to_nid[label] + nid = _make_id(stem, label) + add_node(nid, label, line) + return nid + + def add_edge(src: str, target_label: str, relation: str, line: int, context: str | None = None) -> None: + tgt = ensure_type(target_label, line) + edge = {"source": src, "target": tgt, "relation": relation, + "confidence": "EXTRACTED", "confidence_score": 1.0, + "source_file": str_path, "source_location": f"L{line}", "weight": 1.0} + if context: + edge["context"] = context + edges.append(edge) + + text = _sv_strip_comments(raw) + # Consuming `endclass` (rather than a lookahead) makes each match own its + # terminator, so back-to-back or malformed classes cannot bleed bodies. + class_re = re.compile( + r"\b(?:(interface)\s+)?class\s+(\w+)([^;{]*)\s*;(.*?)\bendclass\b", + re.DOTALL, + ) + for match in class_re.finditer(text): + class_name = match.group(2) + header = match.group(3) or "" + body = match.group(4) or "" + line = line_for(match.start()) + # `#(type T = Payload)` declares `T` as a class type parameter, not a + # referenced type — collect these to skip below. + type_params = frozenset(re.findall(r"\btype\s+(\w+)", header)) + class_nid = _make_id(stem, class_name) + add_node(class_nid, class_name, line) + edges.append({"source": file_nid, "target": class_nid, "relation": "defines", + "confidence": "EXTRACTED", "confidence_score": 1.0, + "source_file": str_path, "source_location": f"L{line}", "weight": 1.0}) + + ext = re.search(r"\bextends\s+(\w+)", header) + if ext: + add_edge(class_nid, ext.group(1), "inherits", line) + impl = re.search(r"\bimplements\s+([^;{]+)", header) + if impl: + for iface_name in _sv_split_type_list(impl.group(1)): + add_edge(class_nid, iface_name.split("#", 1)[0].strip(), "implements", line) + + body_without_functions = re.sub( + r"\bfunction\b.*?\bendfunction\b", + lambda m: "\n" * m.group(0).count("\n"), + body, + flags=re.DOTALL, + ) + for field in re.finditer(r"^\s*([A-Za-z_]\w*(?:\s*#\s*\([^;]+?\))?)\s+\w+\s*;", body_without_functions, re.MULTILINE): + # Count to the start of the type token (group 1), not the match + # start: `^\s*` consumes the leading newline(s), so field.start() + # would resolve to the class's line instead of the field's. + field_line = line + body_without_functions.count("\n", 0, field.start(1)) + for ref_name, role in _sv_collect_type_refs(field.group(1), skip=type_params): + add_edge(class_nid, ref_name, "references", field_line, "generic_arg" if role == "generic_arg" else "field") + + for fm in _SV_FUNC_RE.finditer(body): + return_type, func_name, params = fm.group(1), fm.group(2), fm.group(3) + func_line = line + body.count("\n", 0, fm.start()) + func_nid = _make_id(class_nid, func_name) + add_node(func_nid, func_name, func_line) + edges.append({"source": class_nid, "target": func_nid, "relation": "method", + "confidence": "EXTRACTED", "confidence_score": 1.0, + "source_file": str_path, "source_location": f"L{func_line}", "weight": 1.0}) + for ref_name, role in _sv_collect_type_refs(return_type, skip=type_params): + add_edge(func_nid, ref_name, "references", func_line, "generic_arg" if role == "generic_arg" else "return_type") + for param in _sv_split_type_list(params): + pm = _SV_PARAM_RE.match(param) + if not pm: + continue + for ref_name, role in _sv_collect_type_refs(pm.group(1), skip=type_params): + add_edge(func_nid, ref_name, "references", func_line, "generic_arg" if role == "generic_arg" else "parameter_type") + + def extract_verilog(path: Path) -> dict: - """Extract modules, functions, tasks, package imports, and instantiations from .v/.sv files.""" + """Extract modules, functions, tasks, package imports, instantiations, and + SystemVerilog class semantics (inherits/implements edges, field/parameter/ + return-type references) from .v/.sv files.""" try: import tree_sitter_verilog as tsverilog from tree_sitter import Language, Parser @@ -4865,10 +5066,15 @@ def extract_verilog(path: Path) -> dict: def walk(node, module_nid: str | None = None) -> None: t = node.type + # SystemVerilog class bodies are handled by _augment_systemverilog_semantics + # (regex over source text). Skip their subtrees so in-class methods are not + # double-emitted here — and with the wrong, return-type-derived name. + if t in ("class_declaration", "interface_class_declaration"): + return + if t == "module_declaration": - name_node = node.child_by_field_name("name") - if name_node: - mod_name = _read_text(name_node, source) + mod_name = _sv_first_identifier(_sv_child(node, "module_header"), source) + if mod_name: line = node.start_point[0] + 1 nid = _make_id(stem, mod_name) add_node(nid, mod_name, line) @@ -4877,10 +5083,13 @@ def extract_verilog(path: Path) -> dict: walk(child, nid) return - elif t in ("function_declaration", "function_prototype"): - name_node = node.child_by_field_name("name") - if name_node: - func_name = _read_text(name_node, source) + # `function_prototype` only appears inside class/interface-class bodies + # (skipped above) and nests its name differently; it is intentionally not + # handled here. + elif t == "function_declaration": + fn_body = _sv_child(node, "function_body_declaration") + func_name = _sv_first_identifier(_sv_child(fn_body, "function_identifier"), source) + if func_name: line = node.start_point[0] + 1 parent = module_nid or file_nid nid = _make_id(parent, func_name) @@ -4888,9 +5097,9 @@ def extract_verilog(path: Path) -> dict: add_edge(parent, nid, "contains", line) elif t == "task_declaration": - name_node = node.child_by_field_name("name") - if name_node: - task_name = _read_text(name_node, source) + tk_body = _sv_child(node, "task_body_declaration") + task_name = _sv_first_identifier(_sv_child(tk_body, "task_identifier"), source) + if task_name: line = node.start_point[0] + 1 parent = module_nid or file_nid nid = _make_id(parent, task_name) @@ -4906,14 +5115,18 @@ def extract_verilog(path: Path) -> dict: line = node.start_point[0] + 1 tgt_nid = _make_id(pkg_name) add_node(tgt_nid, pkg_name, line) - src = module_nid or file_nid - add_edge(src, tgt_nid, "imports_from", line) + src_nid = module_nid or file_nid + add_edge(src_nid, tgt_nid, "imports_from", line) - elif t == "module_instantiation": - # module_type instantiates another module - type_node = node.child_by_field_name("module_type") - if type_node and module_nid: - inst_type = _read_text(type_node, source).strip() + elif t in ("module_instantiation", "checker_instantiation"): + # `leaf u_leaf();` parses as checker_instantiation in 1.0.3; + # module_instantiation (when it occurs) exposes a `module_type` field. + # Both reduce to the first identifier under the node — the instantiated + # type, not the instance name (which appears later). + if module_nid: + type_node = node.child_by_field_name("module_type") + inst_type = (_read_text(type_node, source).strip() if type_node + else _sv_first_identifier(node, source)) if inst_type: line = node.start_point[0] + 1 tgt_nid = _make_id(inst_type) @@ -4924,6 +5137,15 @@ def extract_verilog(path: Path) -> dict: walk(child, module_nid) walk(root) + _augment_systemverilog_semantics( + source.decode("utf-8", errors="replace"), + stem, + str_path, + file_nid, + nodes, + edges, + seen_ids, + ) return {"nodes": nodes, "edges": edges} diff --git a/tests/fixtures/sample.sv b/tests/fixtures/sample.sv new file mode 100644 index 00000000..ad8ce4e9 --- /dev/null +++ b/tests/fixtures/sample.sv @@ -0,0 +1,39 @@ +package math_pkg; +endpackage + +interface class Processor; +endclass + +class BaseProcessor; +endclass + +class Payload; +endclass + +class Result #(type T = Payload); + T value; +endclass + +class DataProcessor extends BaseProcessor implements Processor; + Result #(Payload) current; + + function Result #(Payload) build(Payload input); + return current; + endfunction +endclass + +module leaf; +endmodule + +module top; + import math_pkg::*; + + function int add(input int a, input int b); + return a + b; + endfunction + + task tick; + endtask + + leaf u_leaf(); +endmodule diff --git a/tests/test_dart.py b/tests/test_dart.py index f7e22980..094a6f5e 100644 --- a/tests/test_dart.py +++ b/tests/test_dart.py @@ -146,19 +146,57 @@ class TestDart(unittest.TestCase): ) self.assertIsNotNone(configures_injectable) - # Mixin check (Should have global ID "mymixin" and implements edge) + # Mixin check: `with MyMixin` → mixes_in (not implements) ref_mixin = next( ( e for e in edges if e["source"] == user_bloc_node["id"] - and e["target"] == "mymixin" - and e["relation"] == "implements" + and e["target"] == _make_id("MyMixin") + and e["relation"] == "mixes_in" ), None, ) self.assertIsNotNone(ref_mixin) + # Interface check: `implements Disposable` → implements (not mixes_in) + ref_disposable = next( + ( + e + for e in edges + if e["source"] == user_bloc_node["id"] + and e["relation"] == "implements" + and e["target"] == _make_id("Disposable") + ), + None, + ) + self.assertIsNotNone(ref_disposable) + + # Confirm no implements edge targets MyMixin, no mixes_in edge targets Disposable + bad_mixin_implements = next( + ( + e + for e in edges + if e["source"] == user_bloc_node["id"] + and e["target"] == _make_id("MyMixin") + and e["relation"] == "implements" + ), + None, + ) + self.assertIsNone(bad_mixin_implements) + + bad_disposable_mixes_in = next( + ( + e + for e in edges + if e["source"] == user_bloc_node["id"] + and e["target"] == _make_id("Disposable") + and e["relation"] == "mixes_in" + ), + None, + ) + self.assertIsNone(bad_disposable_mixes_in) + # E. Extensions (target class string should be global without stem, source_file is None) ext_node = next((n for n in nodes if n["label"] == "StringExtensions"), None) self.assertIsNotNone(ext_node) diff --git a/tests/test_languages.py b/tests/test_languages.py index c8e2acca..91ab2a17 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -8,7 +8,7 @@ from graphify.extract import ( extract_swift, extract_go, extract_julia, extract_js, extract_fortran, extract_groovy, extract_sln, extract_csproj, extract_razor, extract_dm, extract_dmi, extract_dmm, extract_dmf, - extract_powershell, extract_apex, + extract_powershell, extract_apex, extract_verilog, ) FIXTURES = Path(__file__).parent / "fixtures" @@ -1628,3 +1628,52 @@ def test_apex_no_dangling_edges(): for e in r["edges"]: assert e["source"] in node_ids, f"dangling source in {fixture}: {e}" assert e["target"] in node_ids, f"dangling target in {fixture}: {e}" + + +# -- SystemVerilog ------------------------------------------------------------- + +def test_systemverilog_no_error(): + r = extract_verilog(FIXTURES / "sample.sv") + assert "error" not in r + + +def test_systemverilog_splits_inherits_and_implements(): + r = extract_verilog(FIXTURES / "sample.sv") + assert ("DataProcessor", "BaseProcessor") in _edge_labels(r, "inherits") + assert ("DataProcessor", "Processor") in _edge_labels(r, "implements") + + +def test_systemverilog_field_parameter_return_and_generic_contexts(): + r = extract_verilog(FIXTURES / "sample.sv") + assert ("DataProcessor", "Result") in _edge_labels(r, "references", "field") + assert ("DataProcessor", "Payload") in _edge_labels(r, "references", "generic_arg") + assert ("build", "Payload") in _edge_labels(r, "references", "parameter_type") + assert ("build", "Result") in _edge_labels(r, "references", "return_type") + assert ("build", "Payload") in _edge_labels(r, "references", "generic_arg") + + +def test_systemverilog_does_not_emit_type_parameter_refs(): + r = extract_verilog(FIXTURES / "sample.sv") + assert ("Result", "T") not in _edge_labels(r, "references", "field") + + +def test_systemverilog_preserves_existing_module_extraction(): + r = extract_verilog(FIXTURES / "sample.sv") + labels = set(_labels(r)) + assert {"top", "leaf", "add()", "tick"}.issubset(labels) + assert "imports_from" in _relations(r) + assert "instantiates" in _relations(r) + + +def test_systemverilog_missing_file_returns_empty(): + r = extract_verilog(Path("nonexistent.sv")) + assert r["nodes"] == [] + assert r["edges"] == [] + + +def test_systemverilog_no_dangling_edges(): + r = extract_verilog(FIXTURES / "sample.sv") + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + assert e["source"] in node_ids, f"dangling source: {e}" + assert e["target"] in node_ids, f"dangling target: {e}" diff --git a/uv.lock b/uv.lock index ff119ffa..8010f68f 100644 --- a/uv.lock +++ b/uv.lock @@ -917,21 +917,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/30/09/fe0e3bc32bd33707c519b102fc064ad2a2ce5a1b53e2be38b86936b476b1/cyclonedx_python_lib-11.7.0-py3-none-any.whl", hash = "sha256:02fa4f15ddbba21ac9093039f8137c0d1813af7fe88b760c5dcd3311a8da2178", size = 513041, upload-time = "2026-03-17T15:19:14.369Z" }, ] -[[package]] -name = "datasketch" -version = "1.10.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8d/73/8e9014887f9fca2d785777a0a6186813e4fc7faa24f05fc88c6420624891/datasketch-1.10.0.tar.gz", hash = "sha256:d23aea80ce4c40790ca7a40795659848be92ecc43db80942be26f21e81d24714", size = 91699, upload-time = "2026-04-17T23:06:56.388Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/e7/a94668082e078099eb0161635649510aa887690767b779fffe4bdc479913/datasketch-1.10.0-py3-none-any.whl", hash = "sha256:303dd90cda0948a21abba3aaefc9f8528fa12b8204edc5e1ae8b1d7b750234e7", size = 99914, upload-time = "2026-04-17T23:06:54.39Z" }, -] - [[package]] name = "defusedxml" version = "0.7.1" @@ -1146,12 +1131,13 @@ wheels = [ [[package]] name = "graphifyy" -version = "0.8.35" +version = "0.8.37" source = { editable = "." } dependencies = [ - { name = "datasketch" }, { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, { name = "rapidfuzz" }, { name = "tree-sitter" }, { name = "tree-sitter-bash" }, @@ -1298,7 +1284,6 @@ requires-dist = [ { name = "anthropic", marker = "extra == 'anthropic'" }, { name = "boto3", marker = "extra == 'all'" }, { name = "boto3", marker = "extra == 'bedrock'" }, - { name = "datasketch", specifier = ">=1.6" }, { name = "faster-whisper", marker = "python_full_version >= '3.11' and extra == 'all'" }, { name = "faster-whisper", marker = "python_full_version >= '3.11' and extra == 'video'" }, { name = "graspologic", marker = "python_full_version < '3.13' and extra == 'all'" }, @@ -1314,6 +1299,7 @@ requires-dist = [ { name = "neo4j", marker = "extra == 'all'" }, { name = "neo4j", marker = "extra == 'neo4j'" }, { name = "networkx", specifier = ">=3.4" }, + { name = "numpy", specifier = ">=1.21" }, { name = "numpy", marker = "python_full_version >= '3.13' and extra == 'all'", specifier = ">=2.0" }, { name = "numpy", marker = "python_full_version >= '3.13' and extra == 'svg'", specifier = ">=2.0" }, { name = "openai", marker = "extra == 'all'" }, @@ -4236,19 +4222,12 @@ name = "scipy" version = "1.17.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [