Merge pull request #1267 from TheFedaikin/v8

uv.lock sync, SystemVerilog class-level extraction + Dart mixin fix
This commit is contained in:
Safi
2026-06-11 23:22:27 +01:00
committed by GitHub
5 changed files with 375 additions and 48 deletions
+241 -19
View File
@@ -4541,7 +4541,7 @@ def extract_dart(path: Path) -> dict:
mixin_clean = mixin.split("<")[0].strip()
mixin_nid = _make_id(mixin_clean)
add_node(mixin_nid, mixin_clean, source_file=None)
add_edge(class_nid, mixin_nid, "implements")
add_edge(class_nid, mixin_nid, "mixes_in")
# Map interfaces
for interface in interfaces_list:
@@ -4823,8 +4823,209 @@ def extract_dart(path: Path) -> dict:
return {"nodes": nodes, "edges": edges}
def _sv_first_identifier(node, source: bytes) -> str | None:
"""First `simple_identifier` under node in pre-order, or None.
tree-sitter-verilog 1.0.3 nests declaration names a few levels deep instead
of exposing a `name` field. Scope the search to the right child node (e.g.
`function_identifier`) or this returns the return-type instead of the name.
"""
if node is None:
return None
for child in node.children:
if child.type == "simple_identifier":
return _read_text(child, source)
found = _sv_first_identifier(child, source)
if found:
return found
return None
def _sv_child(node, type_name: str) -> object | None:
if node is None:
return None
for child in node.children:
if child.type == type_name:
return child
return None
_SV_BUILTIN_TYPES = frozenset({
"bit", "logic", "reg", "wire", "int", "integer", "shortint", "longint",
"byte", "time", "real", "shortreal", "void", "string", "type", "event",
"mailbox", "semaphore", "process", "chandle",
})
_SV_NON_TYPE_WORDS = frozenset({
"return", "if", "else", "for", "foreach", "while", "case", "begin", "end",
"function", "task", "class", "endclass", "endfunction", "endtask",
})
# One level of balanced parens (e.g. `Foo #(Bar #(int))`) — bounded so malformed
# input cannot trigger pathological backtracking.
_SV_PARENS_INNER = r"(?:[^()]|\([^()]*\))*"
_SV_PARENS = r"\(" + _SV_PARENS_INNER + r"\)"
_SV_FUNC_RE = re.compile(
r"\bfunction\s+([A-Za-z_]\w*(?:\s*#\s*" + _SV_PARENS + r")?)\s+(\w+)\s*"
r"\((" + _SV_PARENS_INNER + r")\)\s*;",
re.MULTILINE,
)
_SV_PARAM_RE = re.compile(
r"\s*(?:input|output|inout|ref|const\s+ref)?\s*"
r"([A-Za-z_]\w*(?:\s*#\s*" + _SV_PARENS + r")?)\s+\w+"
)
def _sv_strip_comments(text: str) -> str:
text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL)
return re.sub(r"//.*", "", text)
def _sv_split_type_list(text: str) -> list[str]:
parts: list[str] = []
depth = 0
start = 0
for idx, ch in enumerate(text):
if ch == "(":
depth += 1
elif ch == ")":
depth = max(0, depth - 1)
elif ch == "," and depth == 0:
item = text[start:idx].strip()
if item:
parts.append(item)
start = idx + 1
item = text[start:].strip()
if item:
parts.append(item)
return parts
def _sv_collect_type_refs(type_text: str, generic: bool = False,
skip: frozenset[str] = frozenset()) -> list[tuple[str, str]]:
refs: list[tuple[str, str]] = []
text = type_text.strip()
if not text:
return refs
head = re.match(r"([A-Za-z_]\w*)", text)
if head:
name = head.group(1)
# `skip` carries the enclosing class's `#(type T = ...)` parameters so
# they are not mistaken for referenced types.
if name not in _SV_BUILTIN_TYPES and name not in _SV_NON_TYPE_WORDS and name not in skip:
refs.append((name, "generic_arg" if generic else "type"))
params = re.search(r"#\s*\((" + _SV_PARENS_INNER + r")\)", text)
if params:
for arg in _sv_split_type_list(params.group(1)):
refs.extend(_sv_collect_type_refs(arg, generic=True, skip=skip))
return refs
def _augment_systemverilog_semantics(
raw: str,
stem: str,
str_path: str,
file_nid: str,
nodes: list[dict],
edges: list[dict],
seen_ids: set[str],
) -> None:
label_to_nid = {node["label"]: node["id"] for node in nodes}
def line_for(offset: int) -> int:
return raw.count("\n", 0, offset) + 1
def add_node(nid: str, label: str, line: int) -> None:
if nid not in seen_ids:
seen_ids.add(nid)
nodes.append({"id": nid, "label": label, "file_type": "code",
"source_file": str_path, "source_location": f"L{line}",
"confidence_score": 1.0})
label_to_nid[label] = nid
def ensure_type(label: str, line: int) -> str:
if label in label_to_nid:
return label_to_nid[label]
nid = _make_id(stem, label)
add_node(nid, label, line)
return nid
def add_edge(src: str, target_label: str, relation: str, line: int, context: str | None = None) -> None:
tgt = ensure_type(target_label, line)
edge = {"source": src, "target": tgt, "relation": relation,
"confidence": "EXTRACTED", "confidence_score": 1.0,
"source_file": str_path, "source_location": f"L{line}", "weight": 1.0}
if context:
edge["context"] = context
edges.append(edge)
text = _sv_strip_comments(raw)
# Consuming `endclass` (rather than a lookahead) makes each match own its
# terminator, so back-to-back or malformed classes cannot bleed bodies.
class_re = re.compile(
r"\b(?:(interface)\s+)?class\s+(\w+)([^;{]*)\s*;(.*?)\bendclass\b",
re.DOTALL,
)
for match in class_re.finditer(text):
class_name = match.group(2)
header = match.group(3) or ""
body = match.group(4) or ""
line = line_for(match.start())
# `#(type T = Payload)` declares `T` as a class type parameter, not a
# referenced type — collect these to skip below.
type_params = frozenset(re.findall(r"\btype\s+(\w+)", header))
class_nid = _make_id(stem, class_name)
add_node(class_nid, class_name, line)
edges.append({"source": file_nid, "target": class_nid, "relation": "defines",
"confidence": "EXTRACTED", "confidence_score": 1.0,
"source_file": str_path, "source_location": f"L{line}", "weight": 1.0})
ext = re.search(r"\bextends\s+(\w+)", header)
if ext:
add_edge(class_nid, ext.group(1), "inherits", line)
impl = re.search(r"\bimplements\s+([^;{]+)", header)
if impl:
for iface_name in _sv_split_type_list(impl.group(1)):
add_edge(class_nid, iface_name.split("#", 1)[0].strip(), "implements", line)
body_without_functions = re.sub(
r"\bfunction\b.*?\bendfunction\b",
lambda m: "\n" * m.group(0).count("\n"),
body,
flags=re.DOTALL,
)
for field in re.finditer(r"^\s*([A-Za-z_]\w*(?:\s*#\s*\([^;]+?\))?)\s+\w+\s*;", body_without_functions, re.MULTILINE):
# Count to the start of the type token (group 1), not the match
# start: `^\s*` consumes the leading newline(s), so field.start()
# would resolve to the class's line instead of the field's.
field_line = line + body_without_functions.count("\n", 0, field.start(1))
for ref_name, role in _sv_collect_type_refs(field.group(1), skip=type_params):
add_edge(class_nid, ref_name, "references", field_line, "generic_arg" if role == "generic_arg" else "field")
for fm in _SV_FUNC_RE.finditer(body):
return_type, func_name, params = fm.group(1), fm.group(2), fm.group(3)
func_line = line + body.count("\n", 0, fm.start())
func_nid = _make_id(class_nid, func_name)
add_node(func_nid, func_name, func_line)
edges.append({"source": class_nid, "target": func_nid, "relation": "method",
"confidence": "EXTRACTED", "confidence_score": 1.0,
"source_file": str_path, "source_location": f"L{func_line}", "weight": 1.0})
for ref_name, role in _sv_collect_type_refs(return_type, skip=type_params):
add_edge(func_nid, ref_name, "references", func_line, "generic_arg" if role == "generic_arg" else "return_type")
for param in _sv_split_type_list(params):
pm = _SV_PARAM_RE.match(param)
if not pm:
continue
for ref_name, role in _sv_collect_type_refs(pm.group(1), skip=type_params):
add_edge(func_nid, ref_name, "references", func_line, "generic_arg" if role == "generic_arg" else "parameter_type")
def extract_verilog(path: Path) -> dict:
"""Extract modules, functions, tasks, package imports, and instantiations from .v/.sv files."""
"""Extract modules, functions, tasks, package imports, instantiations, and
SystemVerilog class semantics (inherits/implements edges, field/parameter/
return-type references) from .v/.sv files."""
try:
import tree_sitter_verilog as tsverilog
from tree_sitter import Language, Parser
@@ -4865,10 +5066,15 @@ def extract_verilog(path: Path) -> dict:
def walk(node, module_nid: str | None = None) -> None:
t = node.type
# SystemVerilog class bodies are handled by _augment_systemverilog_semantics
# (regex over source text). Skip their subtrees so in-class methods are not
# double-emitted here — and with the wrong, return-type-derived name.
if t in ("class_declaration", "interface_class_declaration"):
return
if t == "module_declaration":
name_node = node.child_by_field_name("name")
if name_node:
mod_name = _read_text(name_node, source)
mod_name = _sv_first_identifier(_sv_child(node, "module_header"), source)
if mod_name:
line = node.start_point[0] + 1
nid = _make_id(stem, mod_name)
add_node(nid, mod_name, line)
@@ -4877,10 +5083,13 @@ def extract_verilog(path: Path) -> dict:
walk(child, nid)
return
elif t in ("function_declaration", "function_prototype"):
name_node = node.child_by_field_name("name")
if name_node:
func_name = _read_text(name_node, source)
# `function_prototype` only appears inside class/interface-class bodies
# (skipped above) and nests its name differently; it is intentionally not
# handled here.
elif t == "function_declaration":
fn_body = _sv_child(node, "function_body_declaration")
func_name = _sv_first_identifier(_sv_child(fn_body, "function_identifier"), source)
if func_name:
line = node.start_point[0] + 1
parent = module_nid or file_nid
nid = _make_id(parent, func_name)
@@ -4888,9 +5097,9 @@ def extract_verilog(path: Path) -> dict:
add_edge(parent, nid, "contains", line)
elif t == "task_declaration":
name_node = node.child_by_field_name("name")
if name_node:
task_name = _read_text(name_node, source)
tk_body = _sv_child(node, "task_body_declaration")
task_name = _sv_first_identifier(_sv_child(tk_body, "task_identifier"), source)
if task_name:
line = node.start_point[0] + 1
parent = module_nid or file_nid
nid = _make_id(parent, task_name)
@@ -4906,14 +5115,18 @@ def extract_verilog(path: Path) -> dict:
line = node.start_point[0] + 1
tgt_nid = _make_id(pkg_name)
add_node(tgt_nid, pkg_name, line)
src = module_nid or file_nid
add_edge(src, tgt_nid, "imports_from", line)
src_nid = module_nid or file_nid
add_edge(src_nid, tgt_nid, "imports_from", line)
elif t == "module_instantiation":
# module_type instantiates another module
type_node = node.child_by_field_name("module_type")
if type_node and module_nid:
inst_type = _read_text(type_node, source).strip()
elif t in ("module_instantiation", "checker_instantiation"):
# `leaf u_leaf();` parses as checker_instantiation in 1.0.3;
# module_instantiation (when it occurs) exposes a `module_type` field.
# Both reduce to the first identifier under the node — the instantiated
# type, not the instance name (which appears later).
if module_nid:
type_node = node.child_by_field_name("module_type")
inst_type = (_read_text(type_node, source).strip() if type_node
else _sv_first_identifier(node, source))
if inst_type:
line = node.start_point[0] + 1
tgt_nid = _make_id(inst_type)
@@ -4924,6 +5137,15 @@ def extract_verilog(path: Path) -> dict:
walk(child, module_nid)
walk(root)
_augment_systemverilog_semantics(
source.decode("utf-8", errors="replace"),
stem,
str_path,
file_nid,
nodes,
edges,
seen_ids,
)
return {"nodes": nodes, "edges": edges}
+39
View File
@@ -0,0 +1,39 @@
package math_pkg;
endpackage
interface class Processor;
endclass
class BaseProcessor;
endclass
class Payload;
endclass
class Result #(type T = Payload);
T value;
endclass
class DataProcessor extends BaseProcessor implements Processor;
Result #(Payload) current;
function Result #(Payload) build(Payload input);
return current;
endfunction
endclass
module leaf;
endmodule
module top;
import math_pkg::*;
function int add(input int a, input int b);
return a + b;
endfunction
task tick;
endtask
leaf u_leaf();
endmodule
+41 -3
View File
@@ -146,19 +146,57 @@ class TestDart(unittest.TestCase):
)
self.assertIsNotNone(configures_injectable)
# Mixin check (Should have global ID "mymixin" and implements edge)
# Mixin check: `with MyMixin` → mixes_in (not implements)
ref_mixin = next(
(
e
for e in edges
if e["source"] == user_bloc_node["id"]
and e["target"] == "mymixin"
and e["relation"] == "implements"
and e["target"] == _make_id("MyMixin")
and e["relation"] == "mixes_in"
),
None,
)
self.assertIsNotNone(ref_mixin)
# Interface check: `implements Disposable` → implements (not mixes_in)
ref_disposable = next(
(
e
for e in edges
if e["source"] == user_bloc_node["id"]
and e["relation"] == "implements"
and e["target"] == _make_id("Disposable")
),
None,
)
self.assertIsNotNone(ref_disposable)
# Confirm no implements edge targets MyMixin, no mixes_in edge targets Disposable
bad_mixin_implements = next(
(
e
for e in edges
if e["source"] == user_bloc_node["id"]
and e["target"] == _make_id("MyMixin")
and e["relation"] == "implements"
),
None,
)
self.assertIsNone(bad_mixin_implements)
bad_disposable_mixes_in = next(
(
e
for e in edges
if e["source"] == user_bloc_node["id"]
and e["target"] == _make_id("Disposable")
and e["relation"] == "mixes_in"
),
None,
)
self.assertIsNone(bad_disposable_mixes_in)
# E. Extensions (target class string should be global without stem, source_file is None)
ext_node = next((n for n in nodes if n["label"] == "StringExtensions"), None)
self.assertIsNotNone(ext_node)
+50 -1
View File
@@ -8,7 +8,7 @@ from graphify.extract import (
extract_swift, extract_go, extract_julia, extract_js, extract_fortran,
extract_groovy, extract_sln, extract_csproj, extract_razor,
extract_dm, extract_dmi, extract_dmm, extract_dmf,
extract_powershell, extract_apex,
extract_powershell, extract_apex, extract_verilog,
)
FIXTURES = Path(__file__).parent / "fixtures"
@@ -1628,3 +1628,52 @@ def test_apex_no_dangling_edges():
for e in r["edges"]:
assert e["source"] in node_ids, f"dangling source in {fixture}: {e}"
assert e["target"] in node_ids, f"dangling target in {fixture}: {e}"
# -- SystemVerilog -------------------------------------------------------------
def test_systemverilog_no_error():
r = extract_verilog(FIXTURES / "sample.sv")
assert "error" not in r
def test_systemverilog_splits_inherits_and_implements():
r = extract_verilog(FIXTURES / "sample.sv")
assert ("DataProcessor", "BaseProcessor") in _edge_labels(r, "inherits")
assert ("DataProcessor", "Processor") in _edge_labels(r, "implements")
def test_systemverilog_field_parameter_return_and_generic_contexts():
r = extract_verilog(FIXTURES / "sample.sv")
assert ("DataProcessor", "Result") in _edge_labels(r, "references", "field")
assert ("DataProcessor", "Payload") in _edge_labels(r, "references", "generic_arg")
assert ("build", "Payload") in _edge_labels(r, "references", "parameter_type")
assert ("build", "Result") in _edge_labels(r, "references", "return_type")
assert ("build", "Payload") in _edge_labels(r, "references", "generic_arg")
def test_systemverilog_does_not_emit_type_parameter_refs():
r = extract_verilog(FIXTURES / "sample.sv")
assert ("Result", "T") not in _edge_labels(r, "references", "field")
def test_systemverilog_preserves_existing_module_extraction():
r = extract_verilog(FIXTURES / "sample.sv")
labels = set(_labels(r))
assert {"top", "leaf", "add()", "tick"}.issubset(labels)
assert "imports_from" in _relations(r)
assert "instantiates" in _relations(r)
def test_systemverilog_missing_file_returns_empty():
r = extract_verilog(Path("nonexistent.sv"))
assert r["nodes"] == []
assert r["edges"] == []
def test_systemverilog_no_dangling_edges():
r = extract_verilog(FIXTURES / "sample.sv")
node_ids = {n["id"] for n in r["nodes"]}
for e in r["edges"]:
assert e["source"] in node_ids, f"dangling source: {e}"
assert e["target"] in node_ids, f"dangling target: {e}"
Generated
+4 -25
View File
@@ -917,21 +917,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/30/09/fe0e3bc32bd33707c519b102fc064ad2a2ce5a1b53e2be38b86936b476b1/cyclonedx_python_lib-11.7.0-py3-none-any.whl", hash = "sha256:02fa4f15ddbba21ac9093039f8137c0d1813af7fe88b760c5dcd3311a8da2178", size = 513041, upload-time = "2026-03-17T15:19:14.369Z" },
]
[[package]]
name = "datasketch"
version = "1.10.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
{ name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/8d/73/8e9014887f9fca2d785777a0a6186813e4fc7faa24f05fc88c6420624891/datasketch-1.10.0.tar.gz", hash = "sha256:d23aea80ce4c40790ca7a40795659848be92ecc43db80942be26f21e81d24714", size = 91699, upload-time = "2026-04-17T23:06:56.388Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/ed/e7/a94668082e078099eb0161635649510aa887690767b779fffe4bdc479913/datasketch-1.10.0-py3-none-any.whl", hash = "sha256:303dd90cda0948a21abba3aaefc9f8528fa12b8204edc5e1ae8b1d7b750234e7", size = 99914, upload-time = "2026-04-17T23:06:54.39Z" },
]
[[package]]
name = "defusedxml"
version = "0.7.1"
@@ -1146,12 +1131,13 @@ wheels = [
[[package]]
name = "graphifyy"
version = "0.8.35"
version = "0.8.37"
source = { editable = "." }
dependencies = [
{ name = "datasketch" },
{ name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
{ name = "rapidfuzz" },
{ name = "tree-sitter" },
{ name = "tree-sitter-bash" },
@@ -1298,7 +1284,6 @@ requires-dist = [
{ name = "anthropic", marker = "extra == 'anthropic'" },
{ name = "boto3", marker = "extra == 'all'" },
{ name = "boto3", marker = "extra == 'bedrock'" },
{ name = "datasketch", specifier = ">=1.6" },
{ name = "faster-whisper", marker = "python_full_version >= '3.11' and extra == 'all'" },
{ name = "faster-whisper", marker = "python_full_version >= '3.11' and extra == 'video'" },
{ name = "graspologic", marker = "python_full_version < '3.13' and extra == 'all'" },
@@ -1314,6 +1299,7 @@ requires-dist = [
{ name = "neo4j", marker = "extra == 'all'" },
{ name = "neo4j", marker = "extra == 'neo4j'" },
{ name = "networkx", specifier = ">=3.4" },
{ name = "numpy", specifier = ">=1.21" },
{ name = "numpy", marker = "python_full_version >= '3.13' and extra == 'all'", specifier = ">=2.0" },
{ name = "numpy", marker = "python_full_version >= '3.13' and extra == 'svg'", specifier = ">=2.0" },
{ name = "openai", marker = "extra == 'all'" },
@@ -4236,19 +4222,12 @@ name = "scipy"
version = "1.17.1"
source = { registry = "https://pypi.org/simple" }
resolution-markers = [
"python_full_version >= '3.14' and sys_platform == 'win32'",
"python_full_version >= '3.14' and sys_platform == 'emscripten'",
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
"python_full_version == '3.13.*' and sys_platform == 'win32'",
"python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'",
"python_full_version == '3.13.*' and sys_platform == 'emscripten'",
"python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'",
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
"python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'",
]
dependencies = [
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" },
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" }
wheels = [