mirror of
https://github.com/safishamsi/graphify.git
synced 2026-09-17 19:15:46 +00:00
Jason Matthew
cc5c54574d
Three independent improvements to extract_corpus_parallel: 1. Token-aware chunking. Replaces `chunk_size=20` static packing with a greedy packer keyed on `token_budget` (default 60_000), grouped by parent directory so related artefacts share a chunk. Pass `token_budget=None` to fall back to fixed-count packing. 2. Optional tiktoken (added to the [kimi] extra). When available, `_estimate_file_tokens` uses cl100k_base for accurate counts; without it, the existing chars/4 heuristic kicks in. Kimi-K2 ships a tiktoken-based tokenizer so estimates against Moonshot are very close to truth. 3. True parallelism. The function name said "parallel" but the body was a sequential for-loop. Now uses ThreadPoolExecutor capped at `max_concurrency` (default 4 — conservative against provider rate limits). `on_chunk_done(idx, total, result)` still fires once per chunk with the original submission idx so progress UIs work unchanged. `max_concurrency=1` skips the pool to preserve sequential semantics. Plus failure tolerance: a chunk raising is now caught, logged to stderr, and the run continues. Other chunks' results merge as normal. On a 162-file repo (~125k words), the same work that took ~36 min sequential under the old code finishes in ~7 min.
70 lines
2.3 KiB
TOML
70 lines
2.3 KiB
TOML
[build-system]
|
|
requires = ["setuptools>=68"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[project]
|
|
name = "graphifyy"
|
|
version = "0.5.6"
|
|
description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, Gemini CLI, Aider, OpenClaw, Factory Droid, Trae, Hermes, Kiro, Google Antigravity) - turn any folder of code, docs, papers, images, or videos into a queryable knowledge graph"
|
|
readme = "README.md"
|
|
license = { file = "LICENSE" }
|
|
keywords = ["claude", "claude-code", "codex", "opencode", "cursor", "gemini", "aider", "kiro", "knowledge-graph", "rag", "graphrag", "obsidian", "community-detection", "tree-sitter", "leiden", "llm"]
|
|
requires-python = ">=3.10"
|
|
dependencies = [
|
|
"networkx",
|
|
"tree-sitter>=0.23.0",
|
|
"tree-sitter-python",
|
|
"tree-sitter-javascript",
|
|
"tree-sitter-typescript",
|
|
"tree-sitter-go",
|
|
"tree-sitter-rust",
|
|
"tree-sitter-java",
|
|
"tree-sitter-c",
|
|
"tree-sitter-cpp",
|
|
"tree-sitter-ruby",
|
|
"tree-sitter-c-sharp",
|
|
"tree-sitter-kotlin",
|
|
"tree-sitter-scala",
|
|
"tree-sitter-php",
|
|
"tree-sitter-swift",
|
|
"tree-sitter-lua",
|
|
"tree-sitter-zig",
|
|
"tree-sitter-powershell",
|
|
"tree-sitter-elixir",
|
|
"tree-sitter-objc",
|
|
"tree-sitter-julia",
|
|
"tree-sitter-verilog",
|
|
]
|
|
|
|
[project.urls]
|
|
Homepage = "https://github.com/safishamsi/graphify"
|
|
Repository = "https://github.com/safishamsi/graphify"
|
|
Issues = "https://github.com/safishamsi/graphify/issues"
|
|
|
|
[project.optional-dependencies]
|
|
mcp = ["mcp"]
|
|
neo4j = ["neo4j"]
|
|
pdf = ["pypdf", "html2text"]
|
|
watch = ["watchdog"]
|
|
svg = ["matplotlib"]
|
|
leiden = ["graspologic; python_version < '3.13'"]
|
|
office = ["python-docx", "openpyxl"]
|
|
video = ["faster-whisper", "yt-dlp"]
|
|
kimi = ["openai", "tiktoken"]
|
|
all = ["mcp", "neo4j", "pypdf", "html2text", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper", "yt-dlp", "matplotlib", "openai", "tiktoken"]
|
|
|
|
[project.scripts]
|
|
graphify = "graphify.__main__:main"
|
|
|
|
[tool.uv]
|
|
# Install via: uv tool install graphifyy
|
|
# Run without installing: uvx graphifyy install
|
|
package = true
|
|
|
|
[tool.setuptools.packages.find]
|
|
where = ["."]
|
|
include = ["graphify*"]
|
|
|
|
[tool.setuptools.package-data]
|
|
graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-aider.md", "skill-copilot.md", "skill-claw.md", "skill-windows.md", "skill-droid.md", "skill-trae.md", "skill-kiro.md", "skill-vscode.md"]
|