feat: add Solidity language extractor

(cherry picked from commit 41a1575e5e)
This commit is contained in:
Abdullah authored and safishamsi committed 2026-09-21 19:13:40 +01:00
1 parent d68d6012bd
commit 9f069eafea
8 files changed
+565 -2

No files matched your search

+1
View File
@@ -42,6 +42,7 @@ _LANG_FAMILY: dict[str, str] = {
**{e: "php" for e in (".php",)},
**{e: "r" for e in (".r",)},
**{e: "cobol" for e in (".cbl", ".cob", ".cobol", ".cpy")},
**{e: "solidity" for e in (".sol",)},
}
+1 -1
View File
@@ -41,7 +41,7 @@ _MANIFEST_PATH = str(out_path("manifest.json"))
_MTIME_COARSE_S = 2.0
_MTIME_SUBSECOND_S = 0.05
CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.vb', '.cbl', '.cob', '.cobol', '.cpy', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.lisp', '.cl', '.lsp', '.asd', '.robot', '.resource'}
CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.vb', '.cbl', '.cob', '.cobol', '.cpy', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.lisp', '.cl', '.lsp', '.asd', '.robot', '.resource', '.sol'}
DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'}
PAPER_EXTENSIONS = {'.pdf'}
IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}
+12
View File
@@ -59,6 +59,10 @@ from graphify.extractors.razor import extract_razor # noqa: F401
from graphify.extractors.robot import extract_robot # noqa: F401
from graphify.extractors.rust import extract_rust # noqa: F401
from graphify.extractors.sln import extract_sln # noqa: F401
from graphify.extractors.solidity import ( # noqa: F401
extract_solidity,
resolve_solidity_type_references,
)
from graphify.extractors.sql import extract_sql # noqa: F401
from graphify.extractors.terraform import extract_terraform, prepare_terraform, resolve_terraform_modules # noqa: F401
from graphify.extractors.verilog import extract_verilog # noqa: F401
@@ -2770,6 +2774,7 @@ _LANG_FAMILY_BY_EXT: dict[str, str] = {
".rs": "rust",
".cbl": "cobol", ".cob": "cobol", ".cobol": "cobol", ".cpy": "cobol",
".r": "r",
".sol": "solidity",
".rb": "ruby", ".rake": "ruby",
".php": "php", ".phtml": "php", ".php3": "php", ".php4": "php",
".php5": "php", ".php7": "php", ".phps": "php",
@@ -5207,6 +5212,11 @@ register_language_resolver(
register_language_resolver(
LanguageResolver("r_sourced_calls", frozenset({".r", ".R"}), resolve_r_sourced_calls)
)
register_language_resolver(
LanguageResolver(
"solidity_type_references", frozenset({".sol"}), resolve_solidity_type_references
)
)
register_language_resolver(
LanguageResolver(
"elixir_import_targets",
@@ -6293,6 +6303,7 @@ _DISPATCH: dict[str, Any] = {
".go": extract_go,
".rs": extract_rust,
".r": extract_r,
".sol": extract_solidity,
".java": extract_java,
".groovy": extract_groovy,
".gradle": extract_groovy,
@@ -6399,6 +6410,7 @@ _DISPATCH: dict[str, Any] = {
_EXTRA_FOR_EXTENSION = {
".vb": "vbnet",
".r": "r",
".sol": "solidity",
".sql": "sql",
".tf": "terraform",
".tfvars": "terraform",
+2
View File
@@ -31,6 +31,7 @@ from graphify.extractors.r import extract_r
from graphify.extractors.razor import extract_razor
from graphify.extractors.rust import extract_rust
from graphify.extractors.sln import extract_sln
from graphify.extractors.solidity import extract_solidity
from graphify.extractors.sql import extract_sql
from graphify.extractors.terraform import extract_terraform
from graphify.extractors.verilog import extract_verilog
@@ -64,6 +65,7 @@ LANGUAGE_EXTRACTORS: dict[str, Callable[[Path], dict]] = {
"razor": extract_razor,
"rust": extract_rust,
"sln": extract_sln,
"solidity": extract_solidity,
"sql": extract_sql,
"terraform": extract_terraform,
"verilog": extract_verilog,
+408
View File
@@ -0,0 +1,408 @@
"""Structural extraction for Solidity source files."""
from __future__ import annotations
from pathlib import Path
from typing import Any
from tree_sitter import Node
from graphify.extractors.base import _file_stem, _make_id, _read_text
_TYPE_DECLARATIONS = {
"contract_declaration": "contract",
"interface_declaration": "interface",
"library_declaration": "library",
}
def _unquote(value: str) -> str:
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"":
return value[1:-1]
return value
def resolve_solidity_type_references(
_per_file: list[dict], all_nodes: list[dict], all_edges: list[dict]
) -> None:
"""Bind inheritance and using references to types in the file or its imports."""
node_by_id = {node.get("id"): node for node in all_nodes}
file_id_by_source = {
str(node["source_file"]): node["id"]
for node in all_nodes
if node.get("source_file")
and node.get("label") == Path(str(node["source_file"])).name
}
source_by_file_id = {nid: source for source, nid in file_id_by_source.items()}
imported_sources: dict[str, set[str]] = {}
for edge in all_edges:
if edge.get("relation") != "imports_from":
continue
source_file = source_by_file_id.get(edge.get("source"))
target_file = source_by_file_id.get(edge.get("target"))
if source_file and target_file:
imported_sources.setdefault(source_file, set()).add(target_file)
types: dict[tuple[str, str], list[str]] = {}
for node in all_nodes:
metadata = node.get("metadata")
if not isinstance(metadata, dict) or metadata.get("kind") not in _TYPE_DECLARATIONS.values():
continue
source_file = node.get("source_file")
if source_file:
types.setdefault((str(source_file), str(node.get("label", ""))), []).append(node["id"])
for edge in all_edges:
context = edge.get("context")
if not isinstance(context, str) or not context.startswith("solidity_type:"):
continue
source_node = node_by_id.get(edge.get("source"))
if not source_node or not source_node.get("source_file"):
continue
source_file = str(source_node["source_file"])
name = context.split(":", 1)[1]
candidates = list(types.get((source_file, name), []))
for imported in imported_sources.get(source_file, set()):
candidates.extend(types.get((imported, name), []))
if len(candidates) == 1:
edge["target"] = candidates[0]
def extract_solidity(path: Path) -> dict:
try:
import tree_sitter_solidity
from tree_sitter import Language, Parser
except ImportError:
return {"nodes": [], "edges": [], "error": "tree-sitter-solidity not installed"}
try:
source = path.read_bytes()
root = Parser(Language(tree_sitter_solidity.language())).parse(source).root_node
except Exception as exc:
return {"nodes": [], "edges": [], "error": f"Solidity grammar failed to load: {exc}"}
source_file = str(path)
stem = _file_stem(path)
file_id = _make_id(source_file)
nodes: list[dict[str, Any]] = []
edges: list[dict[str, Any]] = []
raw_calls: list[dict[str, Any]] = []
seen_ids: set[str] = set()
seen_edges: set[tuple[str, str, str]] = set()
def add_node(
nid: str,
label: str,
node: Node,
*,
kind: str,
source_backed: bool = True,
callable_node: bool = False,
) -> str:
if nid not in seen_ids:
seen_ids.add(nid)
item: dict[str, Any] = {
"id": nid,
"label": label,
"file_type": "code",
"source_location": f"L{node.start_point[0] + 1}",
"metadata": {"language": "solidity", "kind": kind},
}
if source_backed:
item["source_file"] = source_file
if callable_node:
item["_callable"] = True
nodes.append(item)
return nid
def add_edge(
source_id: str,
target_id: str,
relation: str,
node: Node,
*,
context: str | None = None,
target_file: str | None = None,
) -> None:
key = (source_id, target_id, relation)
if not source_id or not target_id or source_id == target_id or key in seen_edges:
return
seen_edges.add(key)
item: dict[str, Any] = {
"source": source_id,
"target": target_id,
"relation": relation,
"confidence": "EXTRACTED",
"source_file": source_file,
"source_location": f"L{node.start_point[0] + 1}",
"weight": 1.0,
}
if context:
item["context"] = context
if target_file:
item["target_file"] = target_file
edges.append(item)
add_node(file_id, path.name, root, kind="file")
def type_stub(name: str, node: Node) -> str:
name = name.split(".")[-1]
nid = _make_id("solidity", "type", name)
add_node(nid, name, node, kind="external_type", source_backed=False)
return nid
for declaration in root.named_children:
if declaration.type != "import_directive":
continue
source_node = declaration.child_by_field_name("source")
if source_node is None:
continue
relative = _unquote(_read_text(source_node, source))
if not relative or relative.startswith(("http:", "https:")):
continue
target = path.parent / relative
add_edge(
file_id,
_make_id(str(target)),
"imports_from",
declaration,
target_file=str(target),
)
for declaration in root.named_children:
type_kind = _TYPE_DECLARATIONS.get(declaration.type)
if type_kind is None:
continue
name_node = declaration.child_by_field_name("name")
body = declaration.child_by_field_name("body")
if name_node is None or body is None:
continue
type_name = _read_text(name_node, source)
type_id = add_node(
_make_id(stem, type_kind, type_name),
type_name,
declaration,
kind=type_kind,
callable_node=type_kind == "contract",
)
add_edge(file_id, type_id, "contains", declaration)
for child in declaration.named_children:
if child.type != "inheritance_specifier":
continue
ancestor = child.child_by_field_name("ancestor")
if ancestor is None:
continue
base_name = _read_text(ancestor, source).split(".")[-1]
add_edge(
type_id,
type_stub(base_name, child),
"inherits",
child,
context=f"solidity_type:{base_name}",
)
functions: dict[tuple[str, int], list[str]] = {}
modifiers: dict[str, str] = {}
reference_targets: dict[tuple[str, int], str] = {}
bodies: list[tuple[Node, str]] = []
pending_modifiers: list[tuple[str, str, Node]] = []
def add_member(
member: Node,
name: str,
label: str,
kind: str,
*,
relation: str = "contains",
signature: str = "",
callable_node: bool = False,
) -> str:
member_id = _make_id(type_id, kind, name, signature or str(member.start_point[0]))
add_node(
member_id,
label,
member,
kind=kind,
callable_node=callable_node,
)
add_edge(type_id, member_id, relation, member)
return member_id
for member in body.named_children:
kind = member.type
if kind == "using_directive":
alias = next(
(
_read_text(child, source)
for child in member.named_children
if child.type in {"type_alias", "identifier", "user_defined_type"}
),
"",
)
if alias:
name = alias.split(".")[-1]
add_edge(
type_id,
type_stub(name, member),
"uses",
member,
context=f"solidity_type:{name}",
)
continue
if kind == "struct_declaration":
member_name_node = member.child_by_field_name("name")
struct_body = member.child_by_field_name("body")
if member_name_node is None:
continue
name = _read_text(member_name_node, source)
struct_id = add_member(member, name, name, "struct")
if struct_body is not None:
for field in struct_body.named_children:
if field.type != "struct_member":
continue
field_name_node = field.child_by_field_name("name")
if field_name_node is None:
continue
field_name = _read_text(field_name_node, source)
field_id = _make_id(struct_id, "field", field_name)
add_node(field_id, field_name, field, kind="field")
add_edge(struct_id, field_id, "contains", field)
continue
if kind == "enum_declaration":
member_name_node = member.child_by_field_name("name")
enum_body = member.child_by_field_name("body")
if member_name_node is None:
continue
name = _read_text(member_name_node, source)
enum_id = add_member(member, name, name, "enum")
if enum_body is not None:
for value in enum_body.named_children:
if value.type != "enum_value":
continue
value_name = _read_text(value, source)
value_id = _make_id(enum_id, value_name)
add_node(value_id, value_name, value, kind="enum_value")
add_edge(enum_id, value_id, "contains", value)
continue
simple_kinds = {
"event_definition": "event",
"error_declaration": "error",
"state_variable_declaration": "state_variable",
}
if kind in simple_kinds:
member_name_node = member.child_by_field_name("name")
if member_name_node is None:
continue
name = _read_text(member_name_node, source)
member_id = add_member(member, name, name, simple_kinds[kind])
arity = sum(child.type in {"event_parameter", "parameter"} for child in member.named_children)
reference_targets[(name, arity)] = member_id
continue
if kind not in {
"function_definition", "constructor_definition", "modifier_definition",
"fallback_receive_definition",
}:
continue
name_node = member.child_by_field_name("name")
if kind == "constructor_definition":
name = "constructor"
callable_kind = "constructor"
elif kind == "fallback_receive_definition":
text = _read_text(member, source).lstrip()
name = "receive" if text.startswith("receive") else "fallback"
callable_kind = name
else:
if name_node is None:
continue
name = _read_text(name_node, source)
callable_kind = "modifier" if kind == "modifier_definition" else "function"
parameters = [
child for child in member.named_children
if child.type in {"parameter", "fallback_receive_parameter"}
]
signature = ",".join(
_read_text(child.child_by_field_name("type"), source)
for child in parameters
if child.child_by_field_name("type") is not None
)
arity = len(parameters)
member_id = add_member(
member,
name,
f"{name}()",
callable_kind,
relation="method",
signature=f"{arity}:{signature}",
callable_node=True,
)
if callable_kind == "modifier":
modifiers[name] = member_id
else:
functions.setdefault((name, arity), []).append(member_id)
member_body = member.child_by_field_name("body")
if member_body is not None:
bodies.append((member_body, member_id))
if callable_kind == "function":
for child in member.named_children:
if child.type != "modifier_invocation":
continue
modifier_name = next(
(
_read_text(item, source)
for item in child.named_children
if item.type == "identifier"
),
"",
)
if modifier_name:
pending_modifiers.append((member_id, modifier_name, child))
for caller_id, modifier_name, modifier_node in pending_modifiers:
target = modifiers.get(modifier_name)
if target:
add_edge(caller_id, target, "uses", modifier_node)
for function_body, caller_id in bodies:
stack = [function_body]
while stack:
node = stack.pop()
if node.type == "call_expression":
callee_node = node.child_by_field_name("function")
if callee_node is not None:
written = _read_text(callee_node, source)
callee = written.split(".")[-1]
arity = sum(child.type == "call_argument" for child in node.named_children)
candidates: list[str] = []
if "." not in written or written.startswith("this."):
candidates = functions.get((callee, arity), [])
if len(candidates) == 1:
add_edge(caller_id, candidates[0], "calls", node)
else:
referenced = reference_targets.get((callee, arity))
if referenced is not None and "." not in written:
add_edge(caller_id, referenced, "references", node)
elif callee and "." in written:
raw_calls.append({
"caller_nid": caller_id,
"callee": callee,
"is_member_call": True,
"language": "solidity",
"source_file": source_file,
"source_location": f"L{node.start_point[0] + 1}",
})
stack.extend(reversed(node.named_children))
clean_edges = [
edge for edge in edges
if edge["source"] in seen_ids
and (edge["target"] in seen_ids or edge["relation"] == "imports_from")
]
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls}
+2 -1
View File
@@ -109,7 +109,8 @@ vbnet = ["tree-sitter-vb-dotnet==0.3.0"]
# No standalone tree-sitter-r wheel exists on PyPI; the language pack bundles the
# R grammar in a prebuilt, offline-compatible wheel. Optional extra (large dep).
r = ["tree-sitter-language-pack==0.11.0"]
all = ["mcp>=1,<3", "starlette>=1.3.1,<2", "neo4j", "falkordb", "pypdf>=6.16.1", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "graspologic-native>=1.3.1,<2; python_version >= '3.13'", "python-docx", "openpyxl", "psycopg[binary]", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.7.4", "matplotlib", "pillow>=12.3.0", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba; python_version < '3.12'", "jieba-py>=0.46.12,<1; python_version >= '3.12'", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal", "tree-sitter-ocaml", "tree-sitter-commonlisp", "robotframework>=4.0", "tree-sitter-vb-dotnet==0.3.0", "tree-sitter-language-pack==0.11.0"]
solidity = ["tree-sitter-solidity==1.2.13"]
all = ["mcp>=1,<3", "starlette>=1.3.1,<2", "neo4j", "falkordb", "pypdf>=6.16.1", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "graspologic-native>=1.3.1,<2; python_version >= '3.13'", "python-docx", "openpyxl", "psycopg[binary]", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.7.4", "matplotlib", "pillow>=12.3.0", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba; python_version < '3.12'", "jieba-py>=0.46.12,<1; python_version >= '3.12'", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal", "tree-sitter-ocaml", "tree-sitter-commonlisp", "robotframework>=4.0", "tree-sitter-vb-dotnet==0.3.0", "tree-sitter-language-pack==0.11.0", "tree-sitter-solidity==1.2.13"]
[project.scripts]
graphify = "graphify.__main__:main"
+9
View File
@@ -0,0 +1,9 @@
pragma solidity ^0.8.20;
contract Sample {
function run() public {
helper();
}
function helper() internal {}
}
+130
View File
@@ -0,0 +1,130 @@
"""Extraction coverage for solidity."""
from __future__ import annotations
import sys
from pathlib import Path
from graphify.extract import extract
FIXTURE = Path(__file__).parent / "fixtures" / "new_languages" / "sample.sol"
def _edge_labels(result: dict, relation: str) -> set[tuple[str, str]]:
labels = {node["id"]: node["label"] for node in result["nodes"]}
return {
(labels.get(edge["source"], edge["source"]), labels.get(edge["target"], edge["target"]))
for edge in result["edges"]
if edge["relation"] == relation
}
def test_solidity_contract_members_and_calls_are_extracted(tmp_path):
source = tmp_path / "Counter.sol"
source.write_text(
"pragma solidity ^0.8.0;\n"
"contract Counter {\n"
" uint public count;\n"
" event Counted(uint value);\n"
" function increment() public { record(); }\n"
" function record() internal {}\n"
"}\n",
encoding="utf-8",
)
result = extract([source], cache_root=tmp_path)
labels = {node["label"] for node in result["nodes"]}
assert {"Counter", "increment()", "record()", "Counted"} <= labels
assert ("increment()", "record()") in _edge_labels(result, "calls")
def test_solidity_types_imports_inheritance_overloads_and_modifiers(tmp_path):
(tmp_path / "Base.sol").write_text(
"contract Base { function baseRun() internal {} }\n", encoding="utf-8"
)
(tmp_path / "Lib.sol").write_text(
"library Lib { function touch(uint value) internal {} }\n", encoding="utf-8"
)
source = tmp_path / "Counter.sol"
source.write_text(
"pragma solidity ^0.8.20;\n"
'import "./Base.sol";\n'
'import "./Lib.sol";\n'
"contract Counter is Base {\n"
" using Lib for uint;\n"
" struct Item { uint value; }\n"
" enum State { On, Off }\n"
" event Changed(uint value);\n"
" error Failed();\n"
" uint public count;\n"
" modifier guarded() { _; }\n"
" constructor() {}\n"
" function run() public guarded { record(); this.record(1); }\n"
" function record() internal {}\n"
" function record(uint value) internal {}\n"
"}\n",
encoding="utf-8",
)
result = extract(
[source, tmp_path / "Base.sol", tmp_path / "Lib.sol"], cache_root=tmp_path
)
labels = [node["label"] for node in result["nodes"]]
assert {
"Counter", "Base", "Lib", "Item", "value", "State", "On", "Off",
"Changed", "Failed", "count", "guarded()", "constructor()", "run()",
} <= set(labels)
assert labels.count("record()") == 2
assert ("Counter", "Base") in _edge_labels(result, "inherits")
assert ("run()", "guarded()") in _edge_labels(result, "uses")
assert len([edge for edge in result["edges"] if edge["relation"] == "imports_from"]) == 2
def test_solidity_fixture_uses_normal_extract_path(tmp_path):
result = extract([FIXTURE], cache_root=tmp_path)
labels = {node["label"] for node in result["nodes"]}
assert {'run()', 'helper()', 'Sample'} <= labels
assert ('run()', 'helper()') in _edge_labels(result, "calls")
def test_solidity_malformed_tail_comments_and_strings_do_not_create_phantoms(tmp_path):
source = tmp_path / 'Broken.sol'
source.write_text('contract Kept { function valid() public {} string constant text = "function Ghost()"; /* function Hidden() {} */', encoding="utf-8")
result = extract([source], cache_root=tmp_path)
labels = {node["label"].casefold() for node in result["nodes"]}
assert 'valid()' in labels
assert labels.isdisjoint({'ghost()', 'hidden()'})
def test_solidity_missing_parser_reports_install_hint(tmp_path, monkeypatch, capsys):
source = tmp_path / "missing.sol"
source.write_text('contract Missing {}\n', encoding="utf-8")
monkeypatch.setitem(sys.modules, 'tree_sitter_solidity', None)
result = extract([source], cache_root=tmp_path)
assert result["nodes"] == []
assert 'pip install "graphifyy[solidity]"' in capsys.readouterr().err