v0.3.17: Julia support, smarter chunking, tree-sitter pin, progress output

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Safi
2026-04-08 22:54:18 +01:00
parent 1896379283
commit 7ff7bd2e8f
13 changed files with 334 additions and 13 deletions
+7
View File
@@ -2,6 +2,13 @@
Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases)
## 0.3.17 (2026-04-08)
- Add: Julia (.jl) support — modules, structs, abstract types, functions, short functions, using/import, call edges, inherits edges via tree-sitter-julia (#98)
- Fix: Semantic extraction chunks now group files by directory so related artifacts land in the same chunk, reducing missed cross-chunk relationships (#65)
- Fix: `tree-sitter>=0.21` now pinned in dependencies — prevents silent empty AST output when older tree-sitter is installed with newer language bindings (#52)
- Add: Progress output every 100 files during AST extraction so large projects don't appear to hang (#52)
## 0.3.16 (2026-04-08)
- Fix: `graphify query`, `serve`, and `benchmark` now work on NetworkX < 3.4 — version-safe shim for `node_link_graph()` at all call sites (#95)
+1 -1
View File
@@ -8,7 +8,7 @@
**An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, OpenClaw, Factory Droid, or Trae - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions.
Fully multimodal. Drop in code, PDFs, markdown, screenshots, diagrams, whiteboard photos, even images in other languages - graphify uses Claude vision to extract concepts and relationships from all of it and connects them into one graph. 19 languages supported via tree-sitter AST (Python, JS, TS, Go, Rust, Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Lua, Zig, PowerShell, Elixir, Objective-C).
Fully multimodal. Drop in code, PDFs, markdown, screenshots, diagrams, whiteboard photos, even images in other languages - graphify uses Claude vision to extract concepts and relationships from all of it and connects them into one graph. 20 languages supported via tree-sitter AST (Python, JS, TS, Go, Rust, Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Lua, Zig, PowerShell, Elixir, Objective-C, Julia).
> Andrej Karpathy keeps a `/raw` folder where he drops papers, tweets, screenshots, and notes. graphify is the answer to that problem - 71.5x fewer tokens per query vs reading the raw files, persistent across sessions, honest about what it found vs guessed.
+1 -1
View File
@@ -17,7 +17,7 @@ class FileType(str, Enum):
_MANIFEST_PATH = "graphify-out/manifest.json"
CODE_EXTENSIONS = {'.py', '.ts', '.js', '.jsx', '.tsx', '.go', '.rs', '.java', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm'}
CODE_EXTENSIONS = {'.py', '.ts', '.js', '.jsx', '.tsx', '.go', '.rs', '.java', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl'}
DOC_EXTENSIONS = {'.md', '.txt', '.rst'}
PAPER_EXTENSIONS = {'.pdf'}
IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}
+220 -1
View File
@@ -1151,6 +1151,218 @@ def extract_swift(path: Path) -> dict:
return _extract_generic(path, _SWIFT_CONFIG)
# ── Julia extractor (custom walk) ────────────────────────────────────────────
def extract_julia(path: Path) -> dict:
"""Extract modules, structs, functions, imports, and calls from a .jl file."""
try:
import tree_sitter_julia as tsjulia
from tree_sitter import Language, Parser
except ImportError:
return {"nodes": [], "edges": [], "error": "tree-sitter-julia not installed"}
try:
language = Language(tsjulia.language())
parser = Parser(language)
source = path.read_bytes()
tree = parser.parse(source)
root = tree.root_node
except Exception as e:
return {"nodes": [], "edges": [], "error": str(e)}
stem = path.stem
str_path = str(path)
nodes: list[dict] = []
edges: list[dict] = []
seen_ids: set[str] = set()
function_bodies: list[tuple[str, object]] = []
def add_node(nid: str, label: str, line: int) -> None:
if nid not in seen_ids:
seen_ids.add(nid)
nodes.append({
"id": nid,
"label": label,
"file_type": "code",
"source_file": str_path,
"source_location": f"L{line}",
})
def add_edge(src: str, tgt: str, relation: str, line: int,
confidence: str = "EXTRACTED", weight: float = 1.0) -> None:
edges.append({
"source": src,
"target": tgt,
"relation": relation,
"confidence": confidence,
"source_file": str_path,
"source_location": f"L{line}",
"weight": weight,
})
file_nid = _make_id(stem)
add_node(file_nid, path.name, 1)
def _func_name_from_signature(sig_node) -> str | None:
"""Extract function name from a Julia signature node (call_expression > identifier)."""
for child in sig_node.children:
if child.type == "call_expression":
callee = child.children[0] if child.children else None
if callee and callee.type == "identifier":
return _read_text(callee, source)
return None
def walk_calls(body_node, func_nid: str) -> None:
if body_node is None:
return
t = body_node.type
if t in ("function_definition", "short_function_definition"):
return
if t == "call_expression" and body_node.children:
callee = body_node.children[0]
# Direct call: foo(...)
if callee.type == "identifier":
callee_name = _read_text(callee, source)
target_nid = _make_id(stem, callee_name)
add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1,
confidence="EXTRACTED")
# Method call: obj.method(...)
elif callee.type == "field_expression" and len(callee.children) >= 3:
method_node = callee.children[-1]
method_name = _read_text(method_node, source)
target_nid = _make_id(stem, method_name)
add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1,
confidence="EXTRACTED")
for child in body_node.children:
walk_calls(child, func_nid)
def walk(node, scope_nid: str) -> None:
t = node.type
# Module
if t == "module_definition":
name_node = next((c for c in node.children if c.type == "identifier"), None)
if name_node:
mod_name = _read_text(name_node, source)
mod_nid = _make_id(stem, mod_name)
line = node.start_point[0] + 1
add_node(mod_nid, mod_name, line)
add_edge(file_nid, mod_nid, "defines", line)
for child in node.children:
walk(child, mod_nid)
return
# Struct (struct / mutable struct — both map to struct_definition in tree-sitter-julia)
if t == "struct_definition":
# type_head may contain: identifier (simple) or binary_expression (Foo <: Bar)
type_head = next((c for c in node.children if c.type == "type_head"), None)
if type_head:
bin_expr = next((c for c in type_head.children if c.type == "binary_expression"), None)
if bin_expr:
# First identifier is the struct name, last is the supertype
identifiers = [c for c in bin_expr.children if c.type == "identifier"]
if identifiers:
struct_name = _read_text(identifiers[0], source)
struct_nid = _make_id(stem, struct_name)
line = node.start_point[0] + 1
add_node(struct_nid, struct_name, line)
add_edge(scope_nid, struct_nid, "defines", line)
if len(identifiers) >= 2:
super_name = _read_text(identifiers[-1], source)
add_edge(struct_nid, _make_id(stem, super_name), "inherits",
line, confidence="EXTRACTED")
else:
name_node = next((c for c in type_head.children if c.type == "identifier"), None)
if name_node:
struct_name = _read_text(name_node, source)
struct_nid = _make_id(stem, struct_name)
line = node.start_point[0] + 1
add_node(struct_nid, struct_name, line)
add_edge(scope_nid, struct_nid, "defines", line)
return
# Abstract type
if t == "abstract_definition":
# type_head > identifier
type_head = next((c for c in node.children if c.type == "type_head"), None)
if type_head:
name_node = next((c for c in type_head.children if c.type == "identifier"), None)
if name_node:
abs_name = _read_text(name_node, source)
abs_nid = _make_id(stem, abs_name)
line = node.start_point[0] + 1
add_node(abs_nid, abs_name, line)
add_edge(scope_nid, abs_nid, "defines", line)
return
# Function: function foo(...) ... end
if t == "function_definition":
sig_node = next((c for c in node.children if c.type == "signature"), None)
if sig_node:
func_name = _func_name_from_signature(sig_node)
if func_name:
func_nid = _make_id(stem, func_name)
line = node.start_point[0] + 1
add_node(func_nid, f"{func_name}()", line)
add_edge(scope_nid, func_nid, "defines", line)
function_bodies.append((func_nid, node))
return
# Short function: foo(x) = expr
if t == "assignment":
lhs = node.children[0] if node.children else None
if lhs and lhs.type == "call_expression" and lhs.children:
callee = lhs.children[0]
if callee.type == "identifier":
func_name = _read_text(callee, source)
func_nid = _make_id(stem, func_name)
line = node.start_point[0] + 1
add_node(func_nid, f"{func_name}()", line)
add_edge(scope_nid, func_nid, "defines", line)
# Only walk the RHS (index 2 after lhs and operator) to avoid self-loops
rhs = node.children[-1] if len(node.children) >= 3 else None
if rhs:
function_bodies.append((func_nid, rhs))
return
# Using / Import
if t in ("using_statement", "import_statement"):
line = node.start_point[0] + 1
for child in node.children:
if child.type == "identifier":
mod_name = _read_text(child, source)
imp_nid = _make_id(mod_name)
add_node(imp_nid, mod_name, line)
add_edge(scope_nid, imp_nid, "imports", line)
elif child.type == "selected_import":
identifiers = [c for c in child.children if c.type == "identifier"]
if identifiers:
pkg_name = _read_text(identifiers[0], source)
pkg_nid = _make_id(pkg_name)
add_node(pkg_nid, pkg_name, line)
add_edge(scope_nid, pkg_nid, "imports", line)
return
for child in node.children:
walk(child, scope_nid)
walk(root, file_nid)
for func_nid, body_node in function_bodies:
# For function_definition nodes, walk children directly to avoid
# the boundary check returning early on the top-level node itself.
# Skip the "signature" child — it contains the function's own call_expression
# which would create a self-loop.
if body_node.type == "function_definition":
for child in body_node.children:
if child.type != "signature":
walk_calls(child, func_nid)
else:
walk_calls(body_node, func_nid)
return {"nodes": nodes, "edges": edges}
# ── Go extractor (custom walk) ────────────────────────────────────────────────
def extract_go(path: Path) -> dict:
@@ -2389,9 +2601,14 @@ def extract(paths: list[Path]) -> dict:
".exs": extract_elixir,
".m": extract_objc,
".mm": extract_objc,
".jl": extract_julia,
}
for path in paths:
total = len(paths)
_PROGRESS_INTERVAL = 100
for i, path in enumerate(paths):
if total >= _PROGRESS_INTERVAL and i % _PROGRESS_INTERVAL == 0 and i > 0:
print(f" AST extraction: {i}/{total} files ({i * 100 // total}%)", flush=True)
extractor = _DISPATCH.get(path.suffix)
if extractor is None:
continue
@@ -2403,6 +2620,8 @@ def extract(paths: list[Path]) -> dict:
if "error" not in result:
save_cached(path, result, root)
per_file.append(result)
if total >= _PROGRESS_INTERVAL:
print(f" AST extraction: {total}/{total} files (100%)", flush=True)
all_nodes: list[dict] = []
all_edges: list[dict] = []
+1 -1
View File
@@ -176,7 +176,7 @@ Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all fil
**Step B1 - Split into chunks**
Load files from `.graphify_uncached.txt`.
Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted.
**Step B2 - Sequential extraction (OpenClaw)**
+1 -1
View File
@@ -179,7 +179,7 @@ Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all fil
**Step B1 - Split into chunks**
Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context).
Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted.
**Step B2 - Dispatch ALL subagents in a single message (Codex)**
+1 -1
View File
@@ -180,7 +180,7 @@ Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all fil
**Step B1 - Split into chunks**
Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context).
Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted.
**Step B2 - Dispatch ALL subagents in a single message (Factory Droid)**
+1 -1
View File
@@ -180,7 +180,7 @@ Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all fil
**Step B1 - Split into chunks**
Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context).
Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted.
**Step B2 - Dispatch ALL subagents in a single message (OpenCode)**
+1 -1
View File
@@ -179,7 +179,7 @@ Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all fil
**Step B1 - Split into chunks**
Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context).
Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted.
**Step B2 - Dispatch ALL subagents using the Agent tool (Trae)**
+1 -1
View File
@@ -182,7 +182,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt
**Step B1 - Split into chunks**
Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context).
Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted.
**Step B2 - Dispatch ALL subagents in a single message**
+3 -2
View File
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "graphifyy"
version = "0.3.16"
version = "0.3.17"
description = "AI coding assistant skill (Claude Code, Codex, OpenCode, OpenClaw) - turn any folder of code, docs, papers, or images into a queryable knowledge graph"
readme = "README.md"
license = { file = "LICENSE" }
@@ -12,7 +12,7 @@ keywords = ["claude", "claude-code", "codex", "opencode", "knowledge-graph", "ra
requires-python = ">=3.10"
dependencies = [
"networkx",
"tree-sitter",
"tree-sitter>=0.21",
"tree-sitter-python",
"tree-sitter-javascript",
"tree-sitter-typescript",
@@ -32,6 +32,7 @@ dependencies = [
"tree-sitter-powershell",
"tree-sitter-elixir",
"tree-sitter-objc",
"tree-sitter-julia",
]
[project.urls]
+33
View File
@@ -0,0 +1,33 @@
module Geometry
using LinearAlgebra
import Base: show
abstract type Shape end
struct Point <: Shape
x::Float64
y::Float64
end
mutable struct Circle <: Shape
center::Point
radius::Float64
end
function area(c::Circle)
return pi * c.radius^2
end
function distance(p1::Point, p2::Point)
return norm([p1.x - p2.x, p1.y - p2.y])
end
perimeter(c::Circle) = 2 * pi * c.radius
function describe(s::Shape)
show(s)
area(s)
end
end
+63 -2
View File
@@ -1,11 +1,11 @@
"""Tests for language extractors: Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Go."""
"""Tests for language extractors: Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Go, Julia."""
from __future__ import annotations
from pathlib import Path
import pytest
from graphify.extract import (
extract_java, extract_c, extract_cpp, extract_ruby,
extract_csharp, extract_kotlin, extract_scala, extract_php,
extract_swift, extract_go,
extract_swift, extract_go, extract_julia,
)
FIXTURES = Path(__file__).parent / "fixtures"
@@ -447,3 +447,64 @@ def test_go_receiver_uses_pkg_scope():
assert server_nodes
# Should NOT contain the file stem "sample" in the type node id
assert "sample" not in server_nodes[0]["id"].split(":")[0]
# ---------------------------------------------------------------------------
# Julia
# ---------------------------------------------------------------------------
def test_julia_finds_module():
r = extract_julia(FIXTURES / "sample.jl")
labels = [n["label"] for n in r["nodes"]]
assert "Geometry" in labels
def test_julia_finds_structs():
r = extract_julia(FIXTURES / "sample.jl")
labels = [n["label"] for n in r["nodes"]]
assert "Point" in labels
assert "Circle" in labels
def test_julia_finds_abstract_type():
r = extract_julia(FIXTURES / "sample.jl")
labels = [n["label"] for n in r["nodes"]]
assert "Shape" in labels
def test_julia_finds_functions():
r = extract_julia(FIXTURES / "sample.jl")
labels = [n["label"] for n in r["nodes"]]
assert any("area" in l for l in labels)
assert any("distance" in l for l in labels)
def test_julia_finds_short_function():
r = extract_julia(FIXTURES / "sample.jl")
labels = [n["label"] for n in r["nodes"]]
assert any("perimeter" in l for l in labels)
def test_julia_finds_imports():
r = extract_julia(FIXTURES / "sample.jl")
import_edges = [e for e in r["edges"] if e["relation"] == "imports"]
assert len(import_edges) >= 1
def test_julia_finds_inherits():
r = extract_julia(FIXTURES / "sample.jl")
inherits = [e for e in r["edges"] if e["relation"] == "inherits"]
assert len(inherits) >= 1
def test_julia_finds_calls():
r = extract_julia(FIXTURES / "sample.jl")
call_edges = [e for e in r["edges"] if e["relation"] == "calls"]
assert len(call_edges) >= 1
def test_julia_no_dangling_edges():
r = extract_julia(FIXTURES / "sample.jl")
node_ids = {n["id"] for n in r["nodes"]}
for e in r["edges"]:
assert e["source"] in node_ids, f"Dangling source: {e}"