diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fd2ef24..884eb91e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.3.17 (2026-04-08) + +- Add: Julia (.jl) support — modules, structs, abstract types, functions, short functions, using/import, call edges, inherits edges via tree-sitter-julia (#98) +- Fix: Semantic extraction chunks now group files by directory so related artifacts land in the same chunk, reducing missed cross-chunk relationships (#65) +- Fix: `tree-sitter>=0.21` now pinned in dependencies — prevents silent empty AST output when older tree-sitter is installed with newer language bindings (#52) +- Add: Progress output every 100 files during AST extraction so large projects don't appear to hang (#52) + ## 0.3.16 (2026-04-08) - Fix: `graphify query`, `serve`, and `benchmark` now work on NetworkX < 3.4 — version-safe shim for `node_link_graph()` at all call sites (#95) diff --git a/README.md b/README.md index 50230b02..9b56a5af 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ **An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, OpenClaw, Factory Droid, or Trae - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. -Fully multimodal. Drop in code, PDFs, markdown, screenshots, diagrams, whiteboard photos, even images in other languages - graphify uses Claude vision to extract concepts and relationships from all of it and connects them into one graph. 19 languages supported via tree-sitter AST (Python, JS, TS, Go, Rust, Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Lua, Zig, PowerShell, Elixir, Objective-C). +Fully multimodal. Drop in code, PDFs, markdown, screenshots, diagrams, whiteboard photos, even images in other languages - graphify uses Claude vision to extract concepts and relationships from all of it and connects them into one graph. 20 languages supported via tree-sitter AST (Python, JS, TS, Go, Rust, Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Lua, Zig, PowerShell, Elixir, Objective-C, Julia). > Andrej Karpathy keeps a `/raw` folder where he drops papers, tweets, screenshots, and notes. graphify is the answer to that problem - 71.5x fewer tokens per query vs reading the raw files, persistent across sessions, honest about what it found vs guessed. diff --git a/graphify/detect.py b/graphify/detect.py index 108544d1..9a5f16e0 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -17,7 +17,7 @@ class FileType(str, Enum): _MANIFEST_PATH = "graphify-out/manifest.json" -CODE_EXTENSIONS = {'.py', '.ts', '.js', '.jsx', '.tsx', '.go', '.rs', '.java', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm'} +CODE_EXTENSIONS = {'.py', '.ts', '.js', '.jsx', '.tsx', '.go', '.rs', '.java', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl'} DOC_EXTENSIONS = {'.md', '.txt', '.rst'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index bec36453..0ea50efb 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -1151,6 +1151,218 @@ def extract_swift(path: Path) -> dict: return _extract_generic(path, _SWIFT_CONFIG) +# ── Julia extractor (custom walk) ──────────────────────────────────────────── + +def extract_julia(path: Path) -> dict: + """Extract modules, structs, functions, imports, and calls from a .jl file.""" + try: + import tree_sitter_julia as tsjulia + from tree_sitter import Language, Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree-sitter-julia not installed"} + + try: + language = Language(tsjulia.language()) + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = path.stem + str_path = str(path) + nodes: list[dict] = [] + edges: list[dict] = [] + seen_ids: set[str] = set() + function_bodies: list[tuple[str, object]] = [] + + def add_node(nid: str, label: str, line: int) -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({ + "id": nid, + "label": label, + "file_type": "code", + "source_file": str_path, + "source_location": f"L{line}", + }) + + def add_edge(src: str, tgt: str, relation: str, line: int, + confidence: str = "EXTRACTED", weight: float = 1.0) -> None: + edges.append({ + "source": src, + "target": tgt, + "relation": relation, + "confidence": confidence, + "source_file": str_path, + "source_location": f"L{line}", + "weight": weight, + }) + + file_nid = _make_id(stem) + add_node(file_nid, path.name, 1) + + def _func_name_from_signature(sig_node) -> str | None: + """Extract function name from a Julia signature node (call_expression > identifier).""" + for child in sig_node.children: + if child.type == "call_expression": + callee = child.children[0] if child.children else None + if callee and callee.type == "identifier": + return _read_text(callee, source) + return None + + def walk_calls(body_node, func_nid: str) -> None: + if body_node is None: + return + t = body_node.type + if t in ("function_definition", "short_function_definition"): + return + if t == "call_expression" and body_node.children: + callee = body_node.children[0] + # Direct call: foo(...) + if callee.type == "identifier": + callee_name = _read_text(callee, source) + target_nid = _make_id(stem, callee_name) + add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1, + confidence="EXTRACTED") + # Method call: obj.method(...) + elif callee.type == "field_expression" and len(callee.children) >= 3: + method_node = callee.children[-1] + method_name = _read_text(method_node, source) + target_nid = _make_id(stem, method_name) + add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1, + confidence="EXTRACTED") + for child in body_node.children: + walk_calls(child, func_nid) + + def walk(node, scope_nid: str) -> None: + t = node.type + + # Module + if t == "module_definition": + name_node = next((c for c in node.children if c.type == "identifier"), None) + if name_node: + mod_name = _read_text(name_node, source) + mod_nid = _make_id(stem, mod_name) + line = node.start_point[0] + 1 + add_node(mod_nid, mod_name, line) + add_edge(file_nid, mod_nid, "defines", line) + for child in node.children: + walk(child, mod_nid) + return + + # Struct (struct / mutable struct — both map to struct_definition in tree-sitter-julia) + if t == "struct_definition": + # type_head may contain: identifier (simple) or binary_expression (Foo <: Bar) + type_head = next((c for c in node.children if c.type == "type_head"), None) + if type_head: + bin_expr = next((c for c in type_head.children if c.type == "binary_expression"), None) + if bin_expr: + # First identifier is the struct name, last is the supertype + identifiers = [c for c in bin_expr.children if c.type == "identifier"] + if identifiers: + struct_name = _read_text(identifiers[0], source) + struct_nid = _make_id(stem, struct_name) + line = node.start_point[0] + 1 + add_node(struct_nid, struct_name, line) + add_edge(scope_nid, struct_nid, "defines", line) + if len(identifiers) >= 2: + super_name = _read_text(identifiers[-1], source) + add_edge(struct_nid, _make_id(stem, super_name), "inherits", + line, confidence="EXTRACTED") + else: + name_node = next((c for c in type_head.children if c.type == "identifier"), None) + if name_node: + struct_name = _read_text(name_node, source) + struct_nid = _make_id(stem, struct_name) + line = node.start_point[0] + 1 + add_node(struct_nid, struct_name, line) + add_edge(scope_nid, struct_nid, "defines", line) + return + + # Abstract type + if t == "abstract_definition": + # type_head > identifier + type_head = next((c for c in node.children if c.type == "type_head"), None) + if type_head: + name_node = next((c for c in type_head.children if c.type == "identifier"), None) + if name_node: + abs_name = _read_text(name_node, source) + abs_nid = _make_id(stem, abs_name) + line = node.start_point[0] + 1 + add_node(abs_nid, abs_name, line) + add_edge(scope_nid, abs_nid, "defines", line) + return + + # Function: function foo(...) ... end + if t == "function_definition": + sig_node = next((c for c in node.children if c.type == "signature"), None) + if sig_node: + func_name = _func_name_from_signature(sig_node) + if func_name: + func_nid = _make_id(stem, func_name) + line = node.start_point[0] + 1 + add_node(func_nid, f"{func_name}()", line) + add_edge(scope_nid, func_nid, "defines", line) + function_bodies.append((func_nid, node)) + return + + # Short function: foo(x) = expr + if t == "assignment": + lhs = node.children[0] if node.children else None + if lhs and lhs.type == "call_expression" and lhs.children: + callee = lhs.children[0] + if callee.type == "identifier": + func_name = _read_text(callee, source) + func_nid = _make_id(stem, func_name) + line = node.start_point[0] + 1 + add_node(func_nid, f"{func_name}()", line) + add_edge(scope_nid, func_nid, "defines", line) + # Only walk the RHS (index 2 after lhs and operator) to avoid self-loops + rhs = node.children[-1] if len(node.children) >= 3 else None + if rhs: + function_bodies.append((func_nid, rhs)) + return + + # Using / Import + if t in ("using_statement", "import_statement"): + line = node.start_point[0] + 1 + for child in node.children: + if child.type == "identifier": + mod_name = _read_text(child, source) + imp_nid = _make_id(mod_name) + add_node(imp_nid, mod_name, line) + add_edge(scope_nid, imp_nid, "imports", line) + elif child.type == "selected_import": + identifiers = [c for c in child.children if c.type == "identifier"] + if identifiers: + pkg_name = _read_text(identifiers[0], source) + pkg_nid = _make_id(pkg_name) + add_node(pkg_nid, pkg_name, line) + add_edge(scope_nid, pkg_nid, "imports", line) + return + + for child in node.children: + walk(child, scope_nid) + + walk(root, file_nid) + + for func_nid, body_node in function_bodies: + # For function_definition nodes, walk children directly to avoid + # the boundary check returning early on the top-level node itself. + # Skip the "signature" child — it contains the function's own call_expression + # which would create a self-loop. + if body_node.type == "function_definition": + for child in body_node.children: + if child.type != "signature": + walk_calls(child, func_nid) + else: + walk_calls(body_node, func_nid) + + return {"nodes": nodes, "edges": edges} + + # ── Go extractor (custom walk) ──────────────────────────────────────────────── def extract_go(path: Path) -> dict: @@ -2389,9 +2601,14 @@ def extract(paths: list[Path]) -> dict: ".exs": extract_elixir, ".m": extract_objc, ".mm": extract_objc, + ".jl": extract_julia, } - for path in paths: + total = len(paths) + _PROGRESS_INTERVAL = 100 + for i, path in enumerate(paths): + if total >= _PROGRESS_INTERVAL and i % _PROGRESS_INTERVAL == 0 and i > 0: + print(f" AST extraction: {i}/{total} files ({i * 100 // total}%)", flush=True) extractor = _DISPATCH.get(path.suffix) if extractor is None: continue @@ -2403,6 +2620,8 @@ def extract(paths: list[Path]) -> dict: if "error" not in result: save_cached(path, result, root) per_file.append(result) + if total >= _PROGRESS_INTERVAL: + print(f" AST extraction: {total}/{total} files (100%)", flush=True) all_nodes: list[dict] = [] all_edges: list[dict] = [] diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index bb04c5e4..c267a7df 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -176,7 +176,7 @@ Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all fil **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. +Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. **Step B2 - Sequential extraction (OpenClaw)** diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index 6c77f632..33d82dfb 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -179,7 +179,7 @@ Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all fil **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). +Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. **Step B2 - Dispatch ALL subagents in a single message (Codex)** diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 66a9f1db..7dfb7381 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -180,7 +180,7 @@ Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all fil **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). +Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. **Step B2 - Dispatch ALL subagents in a single message (Factory Droid)** diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 77e240b5..e03db2b7 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -180,7 +180,7 @@ Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all fil **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). +Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. **Step B2 - Dispatch ALL subagents in a single message (OpenCode)** diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 58f7bb6d..8b2e1ef8 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -179,7 +179,7 @@ Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all fil **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). +Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. **Step B2 - Dispatch ALL subagents using the Agent tool (Trae)** diff --git a/graphify/skill.md b/graphify/skill.md index 2baaa69b..bd744e78 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -182,7 +182,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. **Step B2 - Dispatch ALL subagents in a single message** diff --git a/pyproject.toml b/pyproject.toml index 7cf3cb0c..3ba33a5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.16" +version = "0.3.17" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, OpenClaw) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } @@ -12,7 +12,7 @@ keywords = ["claude", "claude-code", "codex", "opencode", "knowledge-graph", "ra requires-python = ">=3.10" dependencies = [ "networkx", - "tree-sitter", + "tree-sitter>=0.21", "tree-sitter-python", "tree-sitter-javascript", "tree-sitter-typescript", @@ -32,6 +32,7 @@ dependencies = [ "tree-sitter-powershell", "tree-sitter-elixir", "tree-sitter-objc", + "tree-sitter-julia", ] [project.urls] diff --git a/tests/fixtures/sample.jl b/tests/fixtures/sample.jl new file mode 100644 index 00000000..43409f4e --- /dev/null +++ b/tests/fixtures/sample.jl @@ -0,0 +1,33 @@ +module Geometry + +using LinearAlgebra +import Base: show + +abstract type Shape end + +struct Point <: Shape + x::Float64 + y::Float64 +end + +mutable struct Circle <: Shape + center::Point + radius::Float64 +end + +function area(c::Circle) + return pi * c.radius^2 +end + +function distance(p1::Point, p2::Point) + return norm([p1.x - p2.x, p1.y - p2.y]) +end + +perimeter(c::Circle) = 2 * pi * c.radius + +function describe(s::Shape) + show(s) + area(s) +end + +end diff --git a/tests/test_languages.py b/tests/test_languages.py index 551062cf..9784dfa9 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -1,11 +1,11 @@ -"""Tests for language extractors: Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Go.""" +"""Tests for language extractors: Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Go, Julia.""" from __future__ import annotations from pathlib import Path import pytest from graphify.extract import ( extract_java, extract_c, extract_cpp, extract_ruby, extract_csharp, extract_kotlin, extract_scala, extract_php, - extract_swift, extract_go, + extract_swift, extract_go, extract_julia, ) FIXTURES = Path(__file__).parent / "fixtures" @@ -447,3 +447,64 @@ def test_go_receiver_uses_pkg_scope(): assert server_nodes # Should NOT contain the file stem "sample" in the type node id assert "sample" not in server_nodes[0]["id"].split(":")[0] + + +# --------------------------------------------------------------------------- +# Julia +# --------------------------------------------------------------------------- + +def test_julia_finds_module(): + r = extract_julia(FIXTURES / "sample.jl") + labels = [n["label"] for n in r["nodes"]] + assert "Geometry" in labels + + +def test_julia_finds_structs(): + r = extract_julia(FIXTURES / "sample.jl") + labels = [n["label"] for n in r["nodes"]] + assert "Point" in labels + assert "Circle" in labels + + +def test_julia_finds_abstract_type(): + r = extract_julia(FIXTURES / "sample.jl") + labels = [n["label"] for n in r["nodes"]] + assert "Shape" in labels + + +def test_julia_finds_functions(): + r = extract_julia(FIXTURES / "sample.jl") + labels = [n["label"] for n in r["nodes"]] + assert any("area" in l for l in labels) + assert any("distance" in l for l in labels) + + +def test_julia_finds_short_function(): + r = extract_julia(FIXTURES / "sample.jl") + labels = [n["label"] for n in r["nodes"]] + assert any("perimeter" in l for l in labels) + + +def test_julia_finds_imports(): + r = extract_julia(FIXTURES / "sample.jl") + import_edges = [e for e in r["edges"] if e["relation"] == "imports"] + assert len(import_edges) >= 1 + + +def test_julia_finds_inherits(): + r = extract_julia(FIXTURES / "sample.jl") + inherits = [e for e in r["edges"] if e["relation"] == "inherits"] + assert len(inherits) >= 1 + + +def test_julia_finds_calls(): + r = extract_julia(FIXTURES / "sample.jl") + call_edges = [e for e in r["edges"] if e["relation"] == "calls"] + assert len(call_edges) >= 1 + + +def test_julia_no_dangling_edges(): + r = extract_julia(FIXTURES / "sample.jl") + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + assert e["source"] in node_ids, f"Dangling source: {e}"