diff --git a/graphify/detect.py b/graphify/detect.py index 942b542a..a50e03ac 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -18,7 +18,7 @@ class FileType(str, Enum): _MANIFEST_PATH = "graphify-out/manifest.json" -CODE_EXTENSIONS = {'.py', '.ts', '.js', '.jsx', '.tsx', '.mjs', '.ejs', '.go', '.rs', '.java', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.dart', '.v', '.sv'} +CODE_EXTENSIONS = {'.py', '.ts', '.js', '.jsx', '.tsx', '.mjs', '.ejs', '.go', '.rs', '.java', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.dart', '.v', '.sv', '.sql'} DOC_EXTENSIONS = {'.md', '.mdx', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} @@ -169,7 +169,6 @@ def xlsx_to_markdown(path: Path) -> str: ws = wb[sheet_name] rows = [] for row in ws.iter_rows(values_only=True): - # Skip entirely empty rows if all(cell is None for cell in row): continue rows.append([str(cell) if cell is not None else "" for cell in row]) @@ -190,6 +189,91 @@ def xlsx_to_markdown(path: Path) -> str: return "" +def xlsx_extract_structure(path: Path) -> dict: + """Extract structural nodes (sheets, named tables, column headers) from an .xlsx file. + + Returns a nodes/edges dict compatible with the graphify extract pipeline. + Used in addition to xlsx_to_markdown so Claude sees both structure and content. + """ + def _nid(*parts: str) -> str: + return re.sub(r"[^a-z0-9_]", "_", "_".join(p.lower() for p in parts).strip("_")) + + try: + import openpyxl + except ImportError: + return {"nodes": [], "edges": []} + + try: + wb = openpyxl.load_workbook(str(path), read_only=False, data_only=True) + except Exception: + return {"nodes": [], "edges": []} + + stem = _re.sub(r"[^a-z0-9]", "_", path.stem.lower()) + str_path = str(path) + file_nid = _nid(str_path) + nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "document", + "source_file": str_path, "source_location": None}] + edges: list[dict] = [] + seen: set[str] = {file_nid} + + def _add(nid: str, label: str) -> None: + if nid not in seen: + seen.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "document", + "source_file": str_path, "source_location": None}) + + def _edge(src: str, tgt: str, relation: str) -> None: + edges.append({"source": src, "target": tgt, "relation": relation, + "confidence": "EXTRACTED", "source_file": str_path, + "source_location": None, "weight": 1.0}) + + for sheet_name in wb.sheetnames: + ws = wb[sheet_name] + sheet_nid = _nid(stem, sheet_name) + _add(sheet_nid, f"{sheet_name} (sheet)") + _edge(file_nid, sheet_nid, "contains") + + # Named Excel Tables (ListObjects) + if hasattr(ws, "tables"): + for tbl in ws.tables.values(): + tbl_nid = _nid(stem, sheet_name, tbl.name) + _add(tbl_nid, tbl.name) + _edge(sheet_nid, tbl_nid, "contains") + # Column headers from table header row + ref = tbl.ref # e.g. "A1:D10" + if ref: + try: + from openpyxl.utils import range_boundaries + min_col, min_row, max_col, _ = range_boundaries(ref) + header_row = list(ws.iter_rows(min_row=min_row, max_row=min_row, + min_col=min_col, max_col=max_col, + values_only=True)) + if header_row: + for col_name in header_row[0]: + if col_name: + col_nid = _nid(stem, tbl.name, str(col_name)) + _add(col_nid, str(col_name)) + _edge(tbl_nid, col_nid, "contains") + except Exception: + pass + else: + # Fallback: first non-empty row as column headers + for row in ws.iter_rows(max_row=1, values_only=True): + for cell in row: + if cell: + col_nid = _nid(stem, sheet_name, str(cell)) + _add(col_nid, str(cell)) + _edge(sheet_nid, col_nid, "contains") + break + + try: + wb.close() + except Exception: + pass + + return {"nodes": nodes, "edges": edges} + + def convert_office_file(path: Path, out_dir: Path) -> Path | None: """Convert a .docx or .xlsx to a markdown sidecar in out_dir. diff --git a/graphify/extract.py b/graphify/extract.py index 21c1508c..a6951644 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -1710,6 +1710,137 @@ def extract_verilog(path: Path) -> dict: return {"nodes": nodes, "edges": edges} +def extract_sql(path: Path) -> dict: + """Extract tables, views, functions, and relationships from .sql files via tree-sitter.""" + try: + import tree_sitter_sql as tssql + from tree_sitter import Language, Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree_sitter_sql not installed. Run: pip install tree-sitter-sql"} + + try: + language = Language(tssql.language()) + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = re.sub(r"[^a-z0-9]", "_", path.stem.lower()) + str_path = str(path) + file_nid = _make_id(str_path) + nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code", + "source_file": str_path, "source_location": None}] + edges: list[dict] = [] + seen_ids: set[str] = {file_nid} + table_nids: dict[str, str] = {} # name → nid for reference resolution + + def _read(n) -> str: + return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace") + + def _obj_name(n) -> str | None: + for c in n.children: + if c.type == "object_reference": + for cc in c.children: + if cc.type == "identifier": + return _read(cc) + return None + + def _add_node(nid: str, label: str, line: int) -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "code", + "source_file": str_path, "source_location": f"L{line}"}) + edges.append({"source": file_nid, "target": nid, "relation": "contains", + "confidence": "EXTRACTED", "source_file": str_path, + "source_location": f"L{line}", "weight": 1.0}) + + def _add_edge(src: str, tgt: str, relation: str, line: int) -> None: + edges.append({"source": src, "target": tgt, "relation": relation, + "confidence": "EXTRACTED", "source_file": str_path, + "source_location": f"L{line}", "weight": 1.0}) + + def walk(node) -> None: + t = node.type + line = node.start_point[0] + 1 + + if t == "create_table": + name = _obj_name(node) + if name: + nid = _make_id(stem, name) + _add_node(nid, name, line) + table_nids[name.lower()] = nid + # Foreign key REFERENCES + for col in node.children: + if col.type == "column_definitions": + for cd in col.children: + if cd.type != "column_definition": + continue + ref_name: str | None = None + found_ref = False + for cc in cd.children: + if cc.type == "keyword_references": + found_ref = True + elif found_ref and cc.type == "object_reference": + for ccc in cc.children: + if ccc.type == "identifier": + ref_name = _read(ccc) + break + if ref_name: + ref_nid = _make_id(stem, ref_name) + _add_edge(nid, ref_nid, "references", line) + + elif t == "create_view": + name = _obj_name(node) + if name: + nid = _make_id(stem, name) + _add_node(nid, name, line) + table_nids[name.lower()] = nid + # FROM/JOIN table references inside view body + _walk_from_refs(node, nid, line) + + elif t == "create_function": + name = _obj_name(node) + if name: + nid = _make_id(stem, name) + _add_node(nid, f"{name}()", line) + _walk_from_refs(node, nid, line) + + elif t == "create_procedure": + name = _obj_name(node) + if name: + nid = _make_id(stem, name) + _add_node(nid, f"{name}()", line) + _walk_from_refs(node, nid, line) + + for child in node.children: + walk(child) + + def _walk_from_refs(node, caller_nid: str, line: int) -> None: + """Recursively find FROM/JOIN table references inside a node.""" + if node.type in ("from", "join"): + for c in node.children: + if c.type == "relation": + for cc in c.children: + if cc.type == "object_reference": + for ccc in cc.children: + if ccc.type == "identifier": + tbl = _read(ccc) + tbl_nid = _make_id(stem, tbl) + _add_edge(caller_nid, tbl_nid, "reads_from", + c.start_point[0] + 1) + for child in node.children: + _walk_from_refs(child, caller_nid, line) + + for stmt in root.children: + if stmt.type == "statement": + for child in stmt.children: + walk(child) + + return {"nodes": nodes, "edges": edges} + + def extract_lua(path: Path) -> dict: """Extract functions, methods, require() imports, and calls from a .lua file.""" return _extract_generic(path, _LUA_CONFIG) @@ -3359,6 +3490,7 @@ def extract(paths: list[Path], cache_root: Path | None = None) -> dict: ".dart": extract_dart, ".v": extract_verilog, ".sv": extract_verilog, + ".sql": extract_sql, } total = len(paths) diff --git a/pyproject.toml b/pyproject.toml index 836eba1c..3ff870c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,8 @@ leiden = ["graspologic; python_version < '3.13'"] office = ["python-docx", "openpyxl"] video = ["faster-whisper", "yt-dlp"] kimi = ["openai"] -all = ["mcp", "neo4j", "pypdf", "html2text", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper", "yt-dlp", "matplotlib", "openai"] +sql = ["tree-sitter-sql"] +all = ["mcp", "neo4j", "pypdf", "html2text", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper", "yt-dlp", "matplotlib", "openai", "tree-sitter-sql"] [project.scripts] graphify = "graphify.__main__:main" diff --git a/tests/fixtures/sample.sql b/tests/fixtures/sample.sql new file mode 100644 index 00000000..4a59656e --- /dev/null +++ b/tests/fixtures/sample.sql @@ -0,0 +1,19 @@ +CREATE TABLE organizations ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL +); + +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + email TEXT NOT NULL, + org_id INT REFERENCES organizations(id) +); + +CREATE VIEW active_users AS + SELECT * FROM users WHERE active = true; + +CREATE FUNCTION get_user(user_id INT) RETURNS users AS $$ + BEGIN + RETURN QUERY SELECT * FROM users WHERE id = user_id; + END; +$$ LANGUAGE plpgsql; diff --git a/tests/test_multilang.py b/tests/test_multilang.py index 0a67f50b..3264122c 100644 --- a/tests/test_multilang.py +++ b/tests/test_multilang.py @@ -1,9 +1,9 @@ -"""Tests for multi-language AST extraction: JS/TS, Go, Rust.""" +"""Tests for multi-language AST extraction: JS/TS, Go, Rust, SQL.""" from __future__ import annotations import shutil from pathlib import Path import pytest -from graphify.extract import extract_js, extract_go, extract_rust, extract +from graphify.extract import extract_js, extract_go, extract_rust, extract, extract_sql FIXTURES = Path(__file__).parent / "fixtures" @@ -171,3 +171,38 @@ def test_cache_miss_after_file_change(tmp_path): # bar() should appear in the second result labels2 = [n["label"] for n in r2["nodes"]] assert any("bar" in l for l in labels2) + + +# ── SQL ─────────────────────────────────────────────────────────────────────── + +def test_sql_finds_tables(): + r = extract_sql(FIXTURES / "sample.sql") + labels = [n["label"] for n in r["nodes"]] + assert any("users" in l for l in labels) + assert any("organizations" in l for l in labels) + +def test_sql_finds_view(): + r = extract_sql(FIXTURES / "sample.sql") + labels = [n["label"] for n in r["nodes"]] + assert any("active_users" in l for l in labels) + +def test_sql_finds_function(): + r = extract_sql(FIXTURES / "sample.sql") + labels = [n["label"] for n in r["nodes"]] + assert any("get_user" in l for l in labels) + +def test_sql_emits_foreign_key_edge(): + r = extract_sql(FIXTURES / "sample.sql") + relations = {e["relation"] for e in r["edges"]} + assert "references" in relations + +def test_sql_emits_reads_from_edge(): + r = extract_sql(FIXTURES / "sample.sql") + relations = {e["relation"] for e in r["edges"]} + assert "reads_from" in relations + +def test_sql_no_dangling_edges(): + r = extract_sql(FIXTURES / "sample.sql") + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + assert e["source"] in node_ids, f"dangling source: {e['source']}"