From 5904081d7aa27a1abdc1cd6dc2a8136e2c455365 Mon Sep 17 00:00:00 2001 From: Safi Date: Wed, 29 Apr 2026 08:51:53 +0100 Subject: [PATCH] Add Kimi K2.6 backend, fix phantom god nodes (#598), fix concept file_type (#601) Co-Authored-By: Claude Sonnet 4.6 --- README.md | 6 ++ graphify/extract.py | 29 +++++- graphify/llm.py | 210 +++++++++++++++++++++++++++++++++++++++++++ graphify/skill.md | 4 +- graphify/validate.py | 2 +- pyproject.toml | 5 +- 6 files changed, 251 insertions(+), 5 deletions(-) create mode 100644 graphify/llm.py diff --git a/README.md b/README.md index 8ffb17f2..fca5fcee 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,12 @@ dist/ Same syntax as `.gitignore`. You can keep a single `.graphifyignore` at your repo root — patterns work correctly even when graphify is run on a subfolder. +## What's new in v0.5.5 + +- **Kimi K2.6 backend** — `pip install 'graphifyy[kimi]'` then set `MOONSHOT_API_KEY` to route semantic extraction through Kimi K2.6 instead of Claude subagents. 3-6x richer relation extraction at ~3x lower cost. Uses `graphify.llm.extract_corpus_parallel(files, backend="kimi")`. Claude remains the default; Kimi is opt-in. +- **Phantom god node fix (#598)** — member-call callees (`this.logger.log()` → `log`) are no longer cross-file resolved. Previously, any top-level function named `log` anywhere in the corpus would attract hundreds of spurious INFERRED edges from every `Logger.log` call in NestJS/Vue/etc. codebases. Affects all languages: JS/TS, Go, Rust, Swift, Kotlin, Scala, PHP, C++, C#, Zig, Elixir. +- **`concept` file_type fix (#601)** — nodes with `file_type: "concept"` (e.g. tech stack descriptions extracted from Markdown) no longer produce validation warnings. Added `concept` to `VALID_FILE_TYPES`. + ## What's new in v0.5.4 - **SSRF DNS rebinding fix** — `safe_fetch` now patches `socket.getaddrinfo` for the entire duration of each HTTP request so a DNS rebinding attack cannot swap a public IP (returned during validation) for a private one during the actual connection. DNS lookup failures now also raise an error instead of silently skipping the IP check. diff --git a/graphify/extract.py b/graphify/extract.py index 357e4302..aeaca1a5 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -1052,6 +1052,7 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict: if node.type in config.call_types: callee_name: str | None = None + is_member_call: bool = False # Special handling per language if config.ts_module == "tree_sitter_swift": @@ -1061,6 +1062,7 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict: if first.type == "simple_identifier": callee_name = _read_text(first, source) elif first.type == "navigation_expression": + is_member_call = True for child in first.children: if child.type == "navigation_suffix": for sc in child.children: @@ -1073,6 +1075,7 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict: if first.type == "simple_identifier": callee_name = _read_text(first, source) elif first.type == "navigation_expression": + is_member_call = True for child in reversed(first.children): if child.type == "simple_identifier": callee_name = _read_text(child, source) @@ -1084,6 +1087,7 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict: if first.type == "identifier": callee_name = _read_text(first, source) elif first.type == "field_expression": + is_member_call = True field = first.child_by_field_name("field") if field: callee_name = _read_text(field, source) @@ -1103,6 +1107,7 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict: raw = _read_text(child, source) if "." in raw: callee_name = raw.split(".")[-1] + is_member_call = True else: callee_name = raw break @@ -1118,6 +1123,8 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict: if scope_node: callee_name = _read_text(scope_node, source) else: + # member_call_expression: $obj->method() + is_member_call = True name_node = node.child_by_field_name("name") if name_node: callee_name = _read_text(name_node, source) @@ -1128,6 +1135,7 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict: if func_node.type == "identifier": callee_name = _read_text(func_node, source) elif func_node.type in ("field_expression", "qualified_identifier"): + is_member_call = True name = func_node.child_by_field_name("field") or func_node.child_by_field_name("name") if name: callee_name = _read_text(name, source) @@ -1138,6 +1146,7 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict: if func_node.type == "identifier": callee_name = _read_text(func_node, source) elif func_node.type in config.call_accessor_node_types: + is_member_call = True if config.call_accessor_field: attr = func_node.child_by_field_name(config.call_accessor_field) if attr: @@ -1167,6 +1176,7 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict: raw_calls.append({ "caller_nid": caller_nid, "callee": callee_name, + "is_member_call": is_member_call, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", }) @@ -2075,10 +2085,12 @@ def extract_go(path: Path) -> dict: if node.type == "call_expression": func_node = node.child_by_field_name("function") callee_name: str | None = None + is_member_call: bool = False if func_node: if func_node.type == "identifier": callee_name = _read_text(func_node, source) elif func_node.type == "selector_expression": + is_member_call = True field = func_node.child_by_field_name("field") if field: callee_name = _read_text(field, source) @@ -2102,6 +2114,7 @@ def extract_go(path: Path) -> dict: raw_calls.append({ "caller_nid": caller_nid, "callee": callee_name, + "is_member_call": is_member_call, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", }) @@ -2248,10 +2261,12 @@ def extract_rust(path: Path) -> dict: if node.type == "call_expression": func_node = node.child_by_field_name("function") callee_name: str | None = None + is_member_call: bool = False if func_node: if func_node.type == "identifier": callee_name = _read_text(func_node, source) elif func_node.type == "field_expression": + is_member_call = True field = func_node.child_by_field_name("field") if field: callee_name = _read_text(field, source) @@ -2279,6 +2294,7 @@ def extract_rust(path: Path) -> dict: raw_calls.append({ "caller_nid": caller_nid, "callee": callee_name, + "is_member_call": is_member_call, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", }) @@ -2433,7 +2449,9 @@ def extract_zig(path: Path) -> dict: if node.type == "call_expression": fn = node.child_by_field_name("function") if fn: - callee = _read_text(fn, source).split(".")[-1] + fn_text = _read_text(fn, source) + callee = fn_text.split(".")[-1] + is_member_call = "." in fn_text tgt_nid = next((n["id"] for n in nodes if n["label"] in (f"{callee}()", f".{callee}()")), None) if tgt_nid and tgt_nid != caller_nid: @@ -2447,6 +2465,7 @@ def extract_zig(path: Path) -> dict: raw_calls.append({ "caller_nid": caller_nid, "callee": callee, + "is_member_call": is_member_call, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", }) @@ -2611,6 +2630,7 @@ def extract_powershell(path: Path) -> dict: raw_calls.append({ "caller_nid": caller_nid, "callee": cmd_text, + "is_member_call": False, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", }) @@ -3192,8 +3212,10 @@ def extract_elixir(path: Path) -> dict: return break callee_name: str | None = None + is_member_call: bool = False for child in node.children: if child.type == "dot": + is_member_call = True dot_text = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace") parts = dot_text.rstrip(".").split(".") if parts: @@ -3214,6 +3236,7 @@ def extract_elixir(path: Path) -> dict: raw_calls.append({ "caller_nid": caller_nid, "callee": callee_name, + "is_member_call": is_member_call, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", }) @@ -3411,6 +3434,10 @@ def extract(paths: list[Path], cache_root: Path | None = None) -> dict: callee = rc.get("callee", "") if not callee: continue + # Skip member-call callees: obj.log() → "log" has no import evidence + # and collides with any top-level function named "log" in the corpus. + if rc.get("is_member_call"): + continue tgt = global_label_to_nid.get(callee.lower()) caller = rc["caller_nid"] if tgt and tgt != caller and (caller, tgt) not in existing_pairs: diff --git a/graphify/llm.py b/graphify/llm.py new file mode 100644 index 00000000..a9df0e79 --- /dev/null +++ b/graphify/llm.py @@ -0,0 +1,210 @@ +# Direct LLM backend for semantic extraction — supports Claude and Kimi K2.6. +# Used by `graphify . --backend kimi` and the benchmark scripts. +# The default graphify pipeline uses Claude Code subagents via skill.md; +# this module provides a direct API path for non-Claude-Code environments. +from __future__ import annotations + +import json +import os +import time +from pathlib import Path + +BACKENDS: dict[str, dict] = { + "claude": { + "base_url": "https://api.anthropic.com", + "default_model": "claude-sonnet-4-6", + "env_key": "ANTHROPIC_API_KEY", + "pricing": {"input": 3.0, "output": 15.0}, # USD per 1M tokens + }, + "kimi": { + "base_url": "https://api.moonshot.ai/v1", + "default_model": "kimi-k2.6", + "env_key": "MOONSHOT_API_KEY", + "pricing": {"input": 0.74, "output": 4.66}, # USD per 1M tokens + }, +} + +_EXTRACTION_SYSTEM = """\ +You are a graphify semantic extraction agent. Extract a knowledge graph fragment from the files provided. +Output ONLY valid JSON — no explanation, no markdown fences, no preamble. + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, reference) +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain — flag for review, do not omit + +Node ID format: lowercase, only [a-z0-9_], no dots or slashes. +Format: {stem}_{entity} where stem = filename without extension, entity = symbol name (both normalised). + +Output exactly this schema: +{"nodes":[{"id":"stem_entity","label":"Human Readable Name","file_type":"code|document|paper|image|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[],"input_tokens":0,"output_tokens":0} +""" + + +def _read_files(paths: list[Path], root: Path) -> str: + """Return file contents formatted for the extraction prompt.""" + parts: list[str] = [] + for p in paths: + try: + rel = p.relative_to(root) + except ValueError: + rel = p + try: + content = p.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + parts.append(f"=== {rel} ===\n{content[:20000]}") + return "\n\n".join(parts) + + +def _call_openai_compat( + base_url: str, + api_key: str, + model: str, + user_message: str, +) -> dict: + """Call any OpenAI-compatible API (Kimi, OpenAI, etc.) and return parsed JSON.""" + try: + from openai import OpenAI + except ImportError as exc: + raise ImportError( + "Kimi/OpenAI-compatible extraction requires the openai package. " + "Run: pip install openai" + ) from exc + + client = OpenAI(api_key=api_key, base_url=base_url) + resp = client.chat.completions.create( + model=model, + messages=[ + {"role": "system", "content": _EXTRACTION_SYSTEM}, + {"role": "user", "content": user_message}, + ], + max_completion_tokens=8192, + temperature=0, + ) + raw = resp.choices[0].message.content or "{}" + # Strip markdown fences if model adds them despite instructions + if raw.startswith("```"): + raw = raw.split("```", 2)[1] + if raw.startswith("json"): + raw = raw[4:] + raw = raw.rsplit("```", 1)[0] + result = json.loads(raw.strip()) + result["input_tokens"] = resp.usage.prompt_tokens if resp.usage else 0 + result["output_tokens"] = resp.usage.completion_tokens if resp.usage else 0 + result["model"] = model + return result + + +def _call_claude(api_key: str, model: str, user_message: str) -> dict: + """Call Anthropic Claude directly (not via OpenAI compat layer).""" + try: + import anthropic + except ImportError as exc: + raise ImportError( + "Claude direct extraction requires the anthropic package. " + "Run: pip install anthropic" + ) from exc + + client = anthropic.Anthropic(api_key=api_key) + resp = client.messages.create( + model=model, + max_tokens=8192, + system=_EXTRACTION_SYSTEM, + messages=[{"role": "user", "content": user_message}], + ) + raw = resp.content[0].text if resp.content else "{}" + if raw.startswith("```"): + raw = raw.split("```", 2)[1] + if raw.startswith("json"): + raw = raw[4:] + raw = raw.rsplit("```", 1)[0] + result = json.loads(raw.strip()) + result["input_tokens"] = resp.usage.input_tokens if resp.usage else 0 + result["output_tokens"] = resp.usage.output_tokens if resp.usage else 0 + result["model"] = model + return result + + +def extract_files_direct( + files: list[Path], + backend: str = "kimi", + api_key: str | None = None, + model: str | None = None, + root: Path = Path("."), +) -> dict: + """Extract semantic nodes/edges from a list of files using the given backend. + + Returns dict with nodes, edges, hyperedges, input_tokens, output_tokens. + Raises ValueError for unknown backends. Raises ImportError if SDK missing. + """ + if backend not in BACKENDS: + raise ValueError(f"Unknown backend {backend!r}. Available: {sorted(BACKENDS)}") + + cfg = BACKENDS[backend] + key = api_key or os.environ.get(cfg["env_key"], "") + if not key: + raise ValueError( + f"No API key for backend '{backend}'. " + f"Set {cfg['env_key']} or pass api_key=." + ) + mdl = model or cfg["default_model"] + user_msg = _read_files(files, root) + + if backend == "claude": + return _call_claude(key, mdl, user_msg) + else: + return _call_openai_compat(cfg["base_url"], key, mdl, user_msg) + + +def extract_corpus_parallel( + files: list[Path], + backend: str = "kimi", + api_key: str | None = None, + model: str | None = None, + root: Path = Path("."), + chunk_size: int = 20, + on_chunk_done: object = None, +) -> dict: + """Extract a corpus in chunks, merging results. + + on_chunk_done(idx, total, chunk_result) is called after each chunk if provided. + Returns merged dict with nodes, edges, hyperedges, input_tokens, output_tokens. + """ + chunks = [files[i:i + chunk_size] for i in range(0, len(files), chunk_size)] + merged: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} + + for idx, chunk in enumerate(chunks): + t0 = time.time() + result = extract_files_direct(chunk, backend=backend, api_key=api_key, model=model, root=root) + result["elapsed_seconds"] = round(time.time() - t0, 2) + merged["nodes"].extend(result.get("nodes", [])) + merged["edges"].extend(result.get("edges", [])) + merged["hyperedges"].extend(result.get("hyperedges", [])) + merged["input_tokens"] += result.get("input_tokens", 0) + merged["output_tokens"] += result.get("output_tokens", 0) + if callable(on_chunk_done): + on_chunk_done(idx, len(chunks), result) + + return merged + + +def estimate_cost(backend: str, input_tokens: int, output_tokens: int) -> float: + """Estimate USD cost for a given token count using published pricing.""" + if backend not in BACKENDS: + return 0.0 + p = BACKENDS[backend]["pricing"] + return (input_tokens * p["input"] + output_tokens * p["output"]) / 1_000_000 + + +def detect_backend() -> str | None: + """Return the name of whichever backend has an API key set, or None. + + Kimi is checked first (opt-in). Falls back to Claude if ANTHROPIC_API_KEY is set. + Claude is the default for the skill.md subagent pipeline and is never forced here. + """ + if os.environ.get("MOONSHOT_API_KEY"): + return "kimi" + if os.environ.get("ANTHROPIC_API_KEY"): + return "claude" + return None diff --git a/graphify/skill.md b/graphify/skill.md index be1e7dba..09dbc539 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -198,7 +198,9 @@ After transcription: **Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (Claude, costs tokens). +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Kimi K2.6 option:** If `MOONSHOT_API_KEY` is set in the environment, you can run semantic extraction via Kimi instead of Claude subagents. Kimi K2.6 is OpenAI-compatible, extracts 3-6x richer relations, and costs ~3x less per token. To use it: `from graphify.llm import extract_corpus_parallel; result = extract_corpus_parallel(files, backend="kimi")`. Install with `pip install 'graphifyy[kimi]'`. Claude remains the default — Kimi is opt-in via the env var. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/validate.py b/graphify/validate.py index 45139974..5f6bad36 100644 --- a/graphify/validate.py +++ b/graphify/validate.py @@ -1,7 +1,7 @@ # validate extraction JSON against the graphify schema before graph assembly from __future__ import annotations -VALID_FILE_TYPES = {"code", "document", "paper", "image", "rationale"} +VALID_FILE_TYPES = {"code", "document", "paper", "image", "rationale", "concept"} VALID_CONFIDENCES = {"EXTRACTED", "INFERRED", "AMBIGUOUS"} REQUIRED_NODE_FIELDS = {"id", "label", "file_type", "source_file"} REQUIRED_EDGE_FIELDS = {"source", "target", "relation", "confidence", "source_file"} diff --git a/pyproject.toml b/pyproject.toml index a63559b9..c98027d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.5.4" +version = "0.5.5" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, Gemini CLI, Aider, OpenClaw, Factory Droid, Trae, Hermes, Kiro, Google Antigravity) - turn any folder of code, docs, papers, images, or videos into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } @@ -50,7 +50,8 @@ svg = ["matplotlib"] leiden = ["graspologic; python_version < '3.13'"] office = ["python-docx", "openpyxl"] video = ["faster-whisper", "yt-dlp"] -all = ["mcp", "neo4j", "pypdf", "html2text", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper", "yt-dlp", "matplotlib"] +kimi = ["openai"] +all = ["mcp", "neo4j", "pypdf", "html2text", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper", "yt-dlp", "matplotlib", "openai"] [project.scripts] graphify = "graphify.__main__:main"