lee/graphify
1
0
Fork 0
mirror of https://github.com/safishamsi/graphify.git synced 2026-07-12 10:27:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: custom endpoints via OPENAI_BASE_URL/OPENAI_MODEL and ANTHROPIC_BASE_URL/ANTHROPIC_MODEL (#1273)

Browse Source 
Custom OPENAI/ANTHROPIC base-url + model env vars for self-hosted and proxy endpoints. CI green (3.10/3.12).
This commit is contained in:
Timon
2026-06-16 02:41:15 +02:00
committed by GitHub
parent fd470faeee
commit 61836ce0e8
6 changed files with 116 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+7 -1
View File
@@ -398,8 +398,12 @@ These are only needed for **headless / CI extraction** (`graphify extract`). Whe
| Variable | Used for | When required |
|---|---|---|
| `ANTHROPIC_API_KEY` | Claude (Anthropic) backend | `--backend claude` |
| `ANTHROPIC_BASE_URL` | Anthropic-compatible endpoint URL (LiteLLM proxy, gateways, ...) | `--backend claude` (default: `https://api.anthropic.com`) |
| `ANTHROPIC_MODEL` | Model name for the Claude backend — for custom endpoints, use the model name/alias your server exposes | `--backend claude` (default: `claude-sonnet-4-6`) |
| `GEMINI_API_KEY` or `GOOGLE_API_KEY` | Google Gemini backend | `--backend gemini` |
| `OPENAI_API_KEY` | OpenAI or OpenAI-compatible APIs | `--backend openai` |
| `OPENAI_API_KEY` | OpenAI or OpenAI-compatible APIs | `--backend openai` (local servers accept any non-empty value) |
| `OPENAI_BASE_URL` | OpenAI-compatible server URL (llama.cpp, vLLM, LM Studio, ...) | `--backend openai` (default: `https://api.openai.com/v1`) |
| `OPENAI_MODEL` | Model name for the OpenAI backend — for self-hosted servers, use the model name/alias your server exposes (check its `/v1/models` endpoint), e.g. `LFM2.5-8B-A1B-UD-Q4_K_XL` for llama.cpp | `--backend openai` (default: `gpt-4.1-mini`) |
| `DEEPSEEK_API_KEY` | DeepSeek backend | `--backend deepseek` |
| `MOONSHOT_API_KEY` | Kimi Code backend | `--backend kimi` |
| `OLLAMA_BASE_URL` | Ollama local inference URL | `--backend ollama` (default: `http://localhost:11434`) |
@@ -571,6 +575,8 @@ graphify extract ./docs # headless LLM extraction for CI
graphify extract ./docs --backend gemini # explicit backend: gemini, kimi, claude, openai, deepseek, ollama, bedrock, or claude-cli
graphify extract ./docs --backend gemini --model gemini-3.1-pro-preview
graphify extract ./docs --backend ollama # local Ollama (set OLLAMA_BASE_URL / OLLAMA_MODEL) - no API key needed for loopback
OPENAI_BASE_URL=http://localhost:8080/v1 OPENAI_MODEL=my-model graphify extract ./docs --backend openai # any OpenAI-compatible server (llama.cpp, vLLM, LM Studio)
ANTHROPIC_BASE_URL=http://localhost:4000 ANTHROPIC_MODEL=my-model graphify extract ./docs --backend claude # any Anthropic-compatible endpoint (LiteLLM proxy, gateways)
GRAPHIFY_OLLAMA_NUM_CTX=32768 graphify extract ./docs --backend ollama # override KV-cache window (auto-sized by default)
GRAPHIFY_OLLAMA_KEEP_ALIVE=0 graphify extract ./docs --backend ollama # unload model after each chunk (saves VRAM on small GPUs)
graphify extract ./docs --backend bedrock # AWS Bedrock via IAM - no API key, uses AWS credential chain
docs/translations/README.uk-UA.md
+7 -1
View File
@@ -326,8 +326,12 @@ MCP-сервер надає асистенту структурований до
| Змінна | Використання | Коли потрібна |
|---|---|---|
| `ANTHROPIC_API_KEY` | Backend Claude (Anthropic) | `--backend claude` |
| `ANTHROPIC_BASE_URL` | URL Anthropic-сумісного endpoint (LiteLLM proxy, шлюзи, ...) | `--backend claude` (типово: `https://api.anthropic.com`) |
| `ANTHROPIC_MODEL` | Назва моделі для backend Claude — для власних endpoint використовуйте назву/псевдонім моделі вашого сервера | `--backend claude` (типово: `claude-sonnet-4-6`) |
| `GEMINI_API_KEY` або `GOOGLE_API_KEY` | Backend Google Gemini | `--backend gemini` |
| `OPENAI_API_KEY` | OpenAI або OpenAI-сумісні API | `--backend openai` |
| `OPENAI_API_KEY` | OpenAI або OpenAI-сумісні API | `--backend openai` (локальні сервери приймають будь-яке непорожнє значення) |
| `OPENAI_BASE_URL` | URL OpenAI-сумісного сервера (llama.cpp, vLLM, LM Studio, ...) | `--backend openai` (типово: `https://api.openai.com/v1`) |
| `OPENAI_MODEL` | Назва моделі для backend OpenAI — для self-hosted серверів використовуйте назву/псевдонім моделі, яку надає ваш сервер (див. його endpoint `/v1/models`), напр. `LFM2.5-8B-A1B-UD-Q4_K_XL` для llama.cpp | `--backend openai` (типово: `gpt-4.1-mini`) |
| `DEEPSEEK_API_KEY` | Backend DeepSeek | `--backend deepseek` |
| `MOONSHOT_API_KEY` | Backend Kimi Code | `--backend kimi` |
| `OLLAMA_BASE_URL` | URL локального виводу Ollama | `--backend ollama` (типово: `http://localhost:11434`) |
@@ -463,6 +467,8 @@ graphify extract ./docs # headless LLM-витягуван
graphify extract ./docs --backend gemini # явний backend: gemini, kimi, claude, openai, deepseek, ollama, bedrock або claude-cli
graphify extract ./docs --backend gemini --model gemini-3.1-pro-preview
graphify extract ./docs --backend ollama # локальний Ollama (встановіть OLLAMA_BASE_URL / OLLAMA_MODEL) — без API-ключа для loopback
OPENAI_BASE_URL=http://localhost:8080/v1 OPENAI_MODEL=my-model graphify extract ./docs --backend openai # будь-який OpenAI-сумісний сервер (llama.cpp, vLLM, LM Studio)
ANTHROPIC_BASE_URL=http://localhost:4000 ANTHROPIC_MODEL=my-model graphify extract ./docs --backend claude # будь-який Anthropic-сумісний endpoint (LiteLLM proxy, шлюзи)
GRAPHIFY_OLLAMA_NUM_CTX=32768 graphify extract ./docs --backend ollama # перевизначити KV-кеш вікно (автоматично за замовчуванням)
GRAPHIFY_OLLAMA_KEEP_ALIVE=0 graphify extract ./docs --backend ollama # вивантажити модель після кожного шматка (економить VRAM на малих GPU)
graphify extract ./docs --backend bedrock # AWS Bedrock через IAM — без API-ключа, використовує ланцюг облікових даних AWS
graphify/__main__.py
+5
View File
@@ -2182,6 +2182,11 @@ def main() -> None:
print(" --label NAME project label in header")
print(" extract <path> headless full extraction (AST + semantic LLM) for CI/scripts")
print(" --backend B gemini|kimi|claude|openai|deepseek|ollama (default: whichever API key is set)")
print(" openai also reaches self-hosted OpenAI-compatible servers (llama.cpp,")
print(" vLLM, LM Studio): set OPENAI_BASE_URL (e.g. http://localhost:8080/v1)")
print(" and OPENAI_MODEL to the model name your server serves")
print(" claude also reaches custom Anthropic-compatible endpoints (LiteLLM")
print(" proxy, gateways): set ANTHROPIC_BASE_URL and ANTHROPIC_MODEL")
print(" --model M override backend default model")
print(" --mode deep aggressive INFERRED-edge semantic extraction")
print(" --max-workers N AST extraction subprocess count (default: cpu_count)")
graphify/llm.py
+17 -6
View File
@@ -50,8 +50,11 @@ _TOKENIZER = _get_tokenizer()
BACKENDS: dict[str, dict] = {
"claude": {
"base_url": "https://api.anthropic.com",
"default_model": "claude-sonnet-4-6",
# ANTHROPIC_BASE_URL points the backend at any Anthropic-compatible
# server (LiteLLM proxy, gateways, ...); ANTHROPIC_MODEL overrides the
# default model. Mirrors the OPENAI_BASE_URL / OPENAI_MODEL pattern.
"base_url": os.environ.get("ANTHROPIC_BASE_URL", "https://api.anthropic.com"),
"default_model": os.environ.get("ANTHROPIC_MODEL", "claude-sonnet-4-6"),
"env_key": "ANTHROPIC_API_KEY",
"pricing": {"input": 3.0, "output": 15.0}, # USD per 1M tokens
"temperature": 0,
@@ -89,8 +92,12 @@ BACKENDS: dict[str, dict] = {
"vision": True,
},
"openai": {
"base_url": "https://api.openai.com/v1",
"default_model": "gpt-4.1-mini",
# OPENAI_BASE_URL points the backend at any OpenAI-compatible server
# (llama.cpp, vLLM, LM Studio, ...); OPENAI_MODEL overrides the default
# model. GRAPHIFY_OPENAI_MODEL still wins over OPENAI_MODEL when both
# are set (via model_env_key).
"base_url": os.environ.get("OPENAI_BASE_URL", "https://api.openai.com/v1"),
"default_model": os.environ.get("OPENAI_MODEL", "gpt-4.1-mini"),
"env_key": "OPENAI_API_KEY",
"model_env_key": "GRAPHIFY_OPENAI_MODEL",
"pricing": {"input": 0.40, "output": 1.60}, # USD per 1M tokens
@@ -969,7 +976,11 @@ def _call_claude(api_key: str, model: str, user_message: str, max_tokens: int =
except ImportError as exc:
raise ImportError(_backend_pkg_hint("anthropic", "anthropic")) from exc
client = anthropic.Anthropic(api_key=api_key, timeout=_resolve_api_timeout())
client = anthropic.Anthropic(
api_key=api_key,
base_url=BACKENDS["claude"]["base_url"],
timeout=_resolve_api_timeout(),
)
resp = client.messages.create(
model=model,
max_tokens=max_tokens,
@@ -1753,7 +1764,7 @@ def _call_llm(
import anthropic
except ImportError as exc:
raise ImportError(_backend_pkg_hint("anthropic", "anthropic")) from exc
client = anthropic.Anthropic(api_key=key)
client = anthropic.Anthropic(api_key=key, base_url=cfg["base_url"])
resp = client.messages.create(
model=mdl,
max_tokens=max_tokens,
tests/test_anthropic_custom_endpoint.py
+34
View File
@@ -0,0 +1,34 @@
"""Tests for ANTHROPIC_BASE_URL / ANTHROPIC_MODEL overrides on the claude backend.
These env vars point `--backend claude` at any Anthropic-compatible endpoint
(LiteLLM proxy, gateways, ...) without a providers.json entry — mirroring the
OPENAI_BASE_URL / OPENAI_MODEL pattern.
"""
import importlib
from graphify import llm
def test_claude_defaults_without_env(monkeypatch):
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
monkeypatch.delenv("ANTHROPIC_MODEL", raising=False)
reloaded = importlib.reload(llm)
try:
assert reloaded.BACKENDS["claude"]["base_url"] == "https://api.anthropic.com"
assert reloaded.BACKENDS["claude"]["default_model"] == "claude-sonnet-4-6"
finally:
monkeypatch.undo()
importlib.reload(llm)
def test_claude_base_url_and_model_env_override(monkeypatch):
monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://localhost:4000")
monkeypatch.setenv("ANTHROPIC_MODEL", "my-proxied-model")
reloaded = importlib.reload(llm)
try:
assert reloaded.BACKENDS["claude"]["base_url"] == "http://localhost:4000"
assert reloaded.BACKENDS["claude"]["default_model"] == "my-proxied-model"
finally:
monkeypatch.undo()
importlib.reload(llm)
tests/test_openai_custom_endpoint.py
+46
View File
@@ -0,0 +1,46 @@
"""Tests for OPENAI_BASE_URL / OPENAI_MODEL overrides on the openai backend.
These env vars point `--backend openai` at any OpenAI-compatible server
(llama.cpp, vLLM, LM Studio, ...) without a providers.json entry.
"""
import importlib
from graphify import llm
def test_openai_defaults_without_env(monkeypatch):
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
monkeypatch.delenv("OPENAI_MODEL", raising=False)
reloaded = importlib.reload(llm)
try:
assert reloaded.BACKENDS["openai"]["base_url"] == "https://api.openai.com/v1"
assert reloaded.BACKENDS["openai"]["default_model"] == "gpt-4.1-mini"
finally:
monkeypatch.undo()
importlib.reload(llm)
def test_openai_base_url_and_model_env_override(monkeypatch):
monkeypatch.setenv("OPENAI_BASE_URL", "http://localhost:8080/v1")
monkeypatch.setenv("OPENAI_MODEL", "my-local-model")
reloaded = importlib.reload(llm)
try:
assert reloaded.BACKENDS["openai"]["base_url"] == "http://localhost:8080/v1"
assert reloaded.BACKENDS["openai"]["default_model"] == "my-local-model"
finally:
monkeypatch.undo()
importlib.reload(llm)
def test_graphify_openai_model_wins_over_openai_model(monkeypatch):
# model_env_key (GRAPHIFY_OPENAI_MODEL) is resolved at call time and takes
# precedence over the import-time OPENAI_MODEL default.
monkeypatch.setenv("OPENAI_MODEL", "env-default-model")
monkeypatch.setenv("GRAPHIFY_OPENAI_MODEL", "graphify-override-model")
reloaded = importlib.reload(llm)
try:
assert reloaded._default_model_for_backend("openai") == "graphify-override-model"
finally:
monkeypatch.undo()
importlib.reload(llm)