From 82679d06daf6fcd5584560644abd48d5e086a37c Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sat, 24 Oct 2026 21:45:33 +0200 Subject: [PATCH] LLM Config - Dont import LiteLLM until a real config is used (saves memory) (#4596) * LLM - Dont import litellm library until a valid config is entered * LLM - "Test connect" runs in clean subprocess for better memory management * LLM - Add missing model_list module and tests The /settings/llm/models route imports changedetectionio.llm.model_list, which lists provider models over plain HTTP instead of importing litellm. It and its unit tests were left untracked in the earlier commits. Co-Authored-By: Claude Opus 5.5 * LLM - Upgrade litellm to >=1.106.2, add DeepSeek, Groq, Mistral, Together AI and xAI providers (#4597) * Update litellm requirement from >=1.40.0 to >=1.106.2 Supersedes #4595. Co-Authored-By: Claude Opus 5.5 * LLM - Add DeepSeek, Groq, Mistral, Together AI and xAI to the provider list All five serve OpenAI's GET /models, so "Load available models" works for them via model_list without importing litellm. Together's bare-list response and its image/embedding models are handled, and Mistral's {"detail": ...} errors are surfaced. Also re-select Anthropic on page load for saved 'anthropic/claude-...' models, the form the model list returns. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- changedetectionio/blueprint/settings/llm.py | 90 ++------- .../settings/templates/settings_llm_tab.html | 20 +- changedetectionio/llm/client.py | 70 +++++++ changedetectionio/llm/model_list.py | 163 ++++++++++++++++ .../tests/test_llm_api_key_security.py | 28 +-- .../unit/test_llm_completion_isolated.py | 64 +++++++ .../tests/unit/test_llm_model_list.py | 176 ++++++++++++++++++ .../translations/cs/LC_MESSAGES/messages.po | 20 ++ .../translations/de/LC_MESSAGES/messages.po | 20 ++ .../en_GB/LC_MESSAGES/messages.po | 20 ++ .../en_US/LC_MESSAGES/messages.po | 20 ++ .../translations/es/LC_MESSAGES/messages.po | 20 ++ .../translations/fr/LC_MESSAGES/messages.po | 20 ++ .../translations/id/LC_MESSAGES/messages.po | 20 ++ .../translations/it/LC_MESSAGES/messages.po | 20 ++ .../translations/ja/LC_MESSAGES/messages.po | 20 ++ .../translations/ko/LC_MESSAGES/messages.mo | Bin 102732 -> 103111 bytes .../translations/ko/LC_MESSAGES/messages.po | 20 ++ changedetectionio/translations/messages.pot | 22 ++- .../translations/pl/LC_MESSAGES/messages.po | 20 ++ .../pt_BR/LC_MESSAGES/messages.po | 20 ++ .../translations/ru/LC_MESSAGES/messages.po | 20 ++ .../translations/tr/LC_MESSAGES/messages.po | 20 ++ .../translations/uk/LC_MESSAGES/messages.po | 20 ++ .../translations/zh/LC_MESSAGES/messages.po | 20 ++ .../zh_Hant_TW/LC_MESSAGES/messages.mo | Bin 111094 -> 111488 bytes .../zh_Hant_TW/LC_MESSAGES/messages.po | 20 ++ 27 files changed, 880 insertions(+), 93 deletions(-) create mode 100644 changedetectionio/llm/model_list.py create mode 100644 changedetectionio/tests/unit/test_llm_completion_isolated.py create mode 100644 changedetectionio/tests/unit/test_llm_model_list.py diff --git a/changedetectionio/blueprint/settings/llm.py b/changedetectionio/blueprint/settings/llm.py index f4e540383..756630a4c 100644 --- a/changedetectionio/blueprint/settings/llm.py +++ b/changedetectionio/blueprint/settings/llm.py @@ -1,7 +1,4 @@ -import json -import logging import os -import re from flask import Blueprint, jsonify, redirect, url_for, flash from flask_babel import gettext @@ -11,44 +8,6 @@ from changedetectionio.store import ChangeDetectionStore from changedetectionio.auth_decorator import login_optionally_required -class _LiteLLMWarningCapture(logging.Handler): - """Capture warnings emitted on the 'LiteLLM' stdlib logger during a single call. - - litellm.get_valid_models() catches HTTP/auth errors internally, logs a warning, - and returns []. Without capturing that warning we can't tell the user *why* - no models came back (bad key vs. offline vs. genuinely empty model list). - """ - def __init__(self): - super().__init__(level=logging.WARNING) - self.messages = [] - - def emit(self, record): - try: - self.messages.append(record.getMessage()) - except Exception: - pass - - -def _humanize_litellm_error(raw: str) -> str: - # litellm warnings typically look like: - # "Error getting valid models: Failed to get models: { 'error': { 'message': '...' } }" - # Pull the inner provider message when present; otherwise trim the boilerplate. - if not raw: - return raw - m = re.search(r'\{.*\}', raw, re.DOTALL) - if m: - try: - body = json.loads(m.group(0)) - inner = (body.get('error') or {}).get('message') or body.get('message') - if inner: - return inner - except Exception: - pass - cleaned = re.sub(r'^Error getting valid models:\s*', '', raw) - cleaned = re.sub(r'^Failed to get models:\s*', '', cleaned).strip() - return cleaned[:500] - - def construct_llm_blueprint(datastore: ChangeDetectionStore): llm_blueprint = Blueprint('llm', __name__) @@ -91,50 +50,27 @@ def construct_llm_blueprint(datastore: ChangeDetectionStore): "Refusing to send the stored API key to a different endpoint." )}), 400 - _PREFIXES = {'gemini': 'gemini/', 'ollama': 'ollama/', 'openrouter': 'openrouter/', - 'openai_compatible': 'openai/'} - # vLLM / LM Studio / llama.cpp speak OpenAI's wire format — route through litellm's - # 'openai' provider but keep the UI-level name distinct from cloud OpenAI. - _LITELLM_PROVIDER = {'openai_compatible': 'openai'} - prefix = _PREFIXES.get(provider, '') - litellm_provider = _LITELLM_PROVIDER.get(provider, provider) - + # Plain HTTP, not litellm.get_valid_models() - browsing the model list must not + # pull ~200MB of litellm into the process before AI is even configured. + from changedetectionio.llm.model_list import list_models, ModelListError try: - import litellm - logger.debug(f"LLM model list: calling litellm.get_valid_models provider={provider!r} (litellm={litellm_provider!r}) api_base={api_base!r}") - - capture = _LiteLLMWarningCapture() - litellm_logger = logging.getLogger('LiteLLM') - litellm_logger.addHandler(capture) - try: - raw = litellm.get_valid_models( - check_provider_endpoint=True, - custom_llm_provider=litellm_provider, - api_key=api_key or None, - api_base=api_base or None, - ) or [] - finally: - litellm_logger.removeHandler(capture) - - models = sorted({(m if m.startswith(prefix) else prefix + m) for m in raw}) - - if not models and capture.messages: - err = _humanize_litellm_error(capture.messages[-1]) - logger.debug(f"LLM model list: 0 models, surfacing captured litellm warning: {err!r}") - return jsonify({'models': [], 'error': err}), 400 - - logger.debug(f"LLM model list: got {len(models)} models for provider={provider!r}") - return jsonify({'models': models, 'error': None}) + models = list_models(provider, api_key=api_key, api_base=api_base) + except ModelListError as e: + logger.debug(f"LLM model list: provider={provider!r} refused or unreachable: {e}") + return jsonify({'models': [], 'error': str(e)}), 400 except Exception as e: logger.error(f"LLM model list failed for provider={provider!r}: {e}") logger.exception("LLM model list full traceback:") return jsonify({'models': [], 'error': str(e)}), 400 + logger.debug(f"LLM model list: got {len(models)} models for provider={provider!r}") + return jsonify({'models': models, 'error': None}) + @llm_blueprint.route("/test", methods=['POST']) @login_optionally_required def llm_test(): from flask import request - from changedetectionio.llm.client import completion + from changedetectionio.llm.client import completion_isolated from changedetectionio.validate_url import is_llm_api_base_safe # Pull stored config as the fallback, then override with anything the @@ -195,7 +131,9 @@ def construct_llm_blueprint(datastore: ChangeDetectionStore): # a cold-starting or slow-prefilling local model doesn't falsely fail the # test even though the same call would succeed in production (issue #4225). from changedetectionio.llm.evaluator import apply_local_token_multiplier, get_llm_settings, resolve_llm_timeout - text, total_tokens, input_tokens, output_tokens = completion( + # Isolated: trying a provider from the settings page must not leave ~200MB of + # litellm resident in this process (it runs in-process if already loaded). + text, total_tokens, input_tokens, output_tokens = completion_isolated( model=model, messages=[{'role': 'user', 'content': 'Respond with just the word: ready'}], diff --git a/changedetectionio/blueprint/settings/templates/settings_llm_tab.html b/changedetectionio/blueprint/settings/templates/settings_llm_tab.html index 02a90e54b..db2a41b0d 100644 --- a/changedetectionio/blueprint/settings/templates/settings_llm_tab.html +++ b/changedetectionio/blueprint/settings/templates/settings_llm_tab.html @@ -121,11 +121,16 @@ @@ -460,7 +465,8 @@