diff --git a/changedetectionio/blueprint/settings/__init__.py b/changedetectionio/blueprint/settings/__init__.py index e3a75587..1e3ccd3c 100644 --- a/changedetectionio/blueprint/settings/__init__.py +++ b/changedetectionio/blueprint/settings/__init__.py @@ -10,6 +10,7 @@ from flask_babel import gettext from changedetectionio.store import ChangeDetectionStore from changedetectionio.auth_decorator import login_optionally_required +from changedetectionio.model.LLMSettings import LLMSettings def construct_blueprint(datastore: ChangeDetectionStore): @@ -32,25 +33,17 @@ def construct_blueprint(datastore: ChangeDetectionStore): default = deepcopy(datastore.data['settings']) - # Pre-populate LLM sub-form fields from stored config (text fields only — - # PasswordField for api_key is intentionally left blank on GET). - _stored_llm = datastore.data['settings']['application'].get('llm') or {} - default['llm'] = { - 'llm_model': _stored_llm.get('model', ''), - 'llm_api_base': _stored_llm.get('api_base', ''), - 'llm_provider_kind': _stored_llm.get('provider_kind', ''), - 'llm_local_token_multiplier': _stored_llm.get('local_token_multiplier', 5), - 'llm_change_summary_default': datastore.data['settings']['application'].get('llm_change_summary_default', ''), - 'llm_enabled': datastore.data['settings']['application'].get('llm_enabled', True), - 'llm_override_diff_with_summary': datastore.data['settings']['application'].get('llm_override_diff_with_summary', True), - 'llm_restock_use_fallback_extract': datastore.data['settings']['application'].get('llm_restock_use_fallback_extract', True), - 'llm_debug': datastore.data['settings']['application'].get('llm_debug', False), - 'llm_budget_action': datastore.data['settings']['application'].get('llm_budget_action', 'skip_llm'), - 'llm_thinking_budget': str(datastore.data['settings']['application'].get('llm_thinking_budget', 0)), - 'llm_max_summary_tokens': str(datastore.data['settings']['application'].get('llm_max_summary_tokens', 3000)), - 'llm_token_budget_month': _stored_llm.get('token_budget_month', 0), - 'llm_max_input_chars': _stored_llm.get('max_input_chars', 0), - } + # Pre-populate LLM sub-form fields. model_dump(by_alias=True) emits llm_-prefixed + # keys that line up with the WTForms field names. PasswordField for api_key is + # intentionally left blank on GET — submitting a blank value preserves the stored + # key (handled on POST below). SelectField needs its int values as strings. + _stored_llm_settings = LLMSettings.model_validate( + datastore.data['settings']['application'].get('llm') or {} + ) + default['llm'] = _stored_llm_settings.model_dump(by_alias=True) + default['llm']['llm_api_key'] = '' + default['llm']['llm_thinking_budget'] = str(default['llm']['llm_thinking_budget']) + default['llm']['llm_max_summary_tokens'] = str(default['llm']['llm_max_summary_tokens']) if datastore.proxy_list is not None: available_proxies = list(datastore.proxy_list.keys()) @@ -101,82 +94,49 @@ def construct_blueprint(datastore: ChangeDetectionStore): datastore.data['settings']['application'].update(app_update) - # Save LLM config separately under settings.application.llm. - # Token counters (tokens_total_cumulative, tokens_this_month, tokens_month_key) - # are system-managed and must never be overwritten by form submissions. - _LLM_PROTECTED_FIELDS = { - 'tokens_total_cumulative', 'tokens_this_month', 'tokens_month_key', - 'cost_usd_total_cumulative', 'cost_usd_this_month', - } - existing_llm = datastore.data['settings']['application'].get('llm') or {} - preserved_counters = {k: v for k, v in existing_llm.items() if k in _LLM_PROTECTED_FIELDS} - - llm_data = form.data.get('llm') or {} - - # PasswordField never re-populates its value on GET, so the submitted value - # is only non-empty when the user explicitly typed a new key. - # If blank, preserve the existing key so a settings save doesn't accidentally clear it. - submitted_api_key = (llm_data.get('llm_api_key') or '').strip() - effective_api_key = submitted_api_key if submitted_api_key else existing_llm.get('api_key', '') - - # Application-level LLM settings (survive provider changes) - datastore.data['settings']['application']['llm_change_summary_default'] = ( - llm_data.get('llm_change_summary_default') or '' - ).strip() - datastore.data['settings']['application']['llm_enabled'] = ( - bool(llm_data.get('llm_enabled', True)) - ) - datastore.data['settings']['application']['llm_override_diff_with_summary'] = ( - bool(llm_data.get('llm_override_diff_with_summary', True)) - ) - datastore.data['settings']['application']['llm_restock_use_fallback_extract'] = ( - bool(llm_data.get('llm_restock_use_fallback_extract', True)) - ) - datastore.data['settings']['application']['llm_debug'] = ( - bool(llm_data.get('llm_debug', False)) - ) - datastore.data['settings']['application']['llm_budget_action'] = ( - llm_data.get('llm_budget_action') or 'skip_llm' - ) - datastore.data['settings']['application']['llm_thinking_budget'] = ( - int(llm_data.get('llm_thinking_budget') or 0) - ) - datastore.data['settings']['application']['llm_max_summary_tokens'] = ( - int(llm_data.get('llm_max_summary_tokens') or 3000) + # LLM config now lives under settings.application.llm.* (post update_31). + # Strategy: hydrate the stored dict into LLMSettings (preserves system + # counters and any forward-compat extras via extra='allow'), then merge + # the form input over it. Pydantic's populate_by_name accepts both the + # stripped names (existing storage) and the llm_-prefixed aliases + # (form field names) in the same call. + existing_llm = LLMSettings.model_validate( + datastore.data['settings']['application'].get('llm') or {} ) - # Monthly token budget — only save if env var is not set - import os as _os - if not _os.getenv('LLM_TOKEN_BUDGET_MONTH', '').strip(): - _budget = llm_data.get('llm_token_budget_month') or 0 - existing_llm['token_budget_month'] = int(_budget) if _budget else 0 + llm_form_input = dict(form.data.get('llm') or {}) - # Max input chars — only save if env var is not set - if not _os.getenv('LLM_MAX_INPUT_CHARS', '').strip(): - _max_chars = llm_data.get('llm_max_input_chars') or 0 - existing_llm['max_input_chars'] = int(_max_chars) if _max_chars else 0 + # PasswordField never re-renders, so a blank submitted value means + # "keep stored key" — drop it from the merge. + if not (llm_form_input.get('llm_api_key') or '').strip(): + llm_form_input.pop('llm_api_key', None) - llm_config = { - 'model': (llm_data.get('llm_model') or '').strip(), - 'api_key': effective_api_key, - 'api_base': (llm_data.get('llm_api_base') or '').strip(), - # Identifies a self-hosted OpenAI-compatible endpoint so reasoning-friendly - # token caps can be applied conditionally (cloud-LLM defaults stay tight). - 'provider_kind': (llm_data.get('llm_provider_kind') or '').strip(), - 'local_token_multiplier': int(llm_data.get('llm_local_token_multiplier') or 5), - 'token_budget_month': existing_llm.get('token_budget_month', 0), - 'max_input_chars': existing_llm.get('max_input_chars', 0), - **preserved_counters, - } - # Only store if a model is set - if llm_config['model']: - datastore.data['settings']['application']['llm'] = llm_config - else: - # Remove model config but retain counters for historical record - if preserved_counters: - datastore.data['settings']['application']['llm'] = preserved_counters + # Env-var overrides make these fields read-only in the UI — ignore form input. + if os.getenv('LLM_TOKEN_BUDGET_MONTH', '').strip(): + llm_form_input.pop('llm_token_budget_month', None) + if os.getenv('LLM_MAX_INPUT_CHARS', '').strip(): + llm_form_input.pop('llm_max_input_chars', None) + + # System-managed counters must never come from the form. + for protected in LLMSettings.PROTECTED_FIELDS: + llm_form_input.pop(protected, None) + + # by_alias=True so existing values appear under the same key shape as the + # form input (llm_*). Without this, extra='allow' would store the + # stripped-name version as a model_extra that shadows the field value on + # model_dump() — even though .attribute access reads the alias correctly. + merged = LLMSettings.model_validate({**existing_llm.model_dump(by_alias=True), **llm_form_input}) + + # Clearing the model field drops the saved provider config but retains + # any historical counter values (so monthly usage charts don't reset). + if not merged.model.strip(): + counters = {k: getattr(merged, k) for k in LLMSettings.PROTECTED_FIELDS} + if any(counters.values()): + datastore.data['settings']['application']['llm'] = counters else: datastore.data['settings']['application'].pop('llm', None) + else: + datastore.data['settings']['application']['llm'] = merged.model_dump() # Handle dynamic worker count adjustment old_worker_count = datastore.data['settings']['requests'].get('workers', 1) diff --git a/changedetectionio/blueprint/settings/llm.py b/changedetectionio/blueprint/settings/llm.py index af3ae586..c07da003 100644 --- a/changedetectionio/blueprint/settings/llm.py +++ b/changedetectionio/blueprint/settings/llm.py @@ -193,7 +193,7 @@ def construct_llm_blueprint(datastore: ChangeDetectionStore): # via LLM_TIMEOUT). A shorter test-only timeout falsely fails on cold-starting # cloud reasoning models (e.g. ollama.com hosting qwen3.5:397b takes ~60s on # first hit) even though the same call succeeds in production. - from changedetectionio.llm.evaluator import apply_local_token_multiplier + from changedetectionio.llm.evaluator import apply_local_token_multiplier, get_llm_settings text, total_tokens, input_tokens, output_tokens = completion( model=model, messages=[{'role': 'user', 'content': @@ -201,7 +201,7 @@ def construct_llm_blueprint(datastore: ChangeDetectionStore): api_key=llm_cfg.get('api_key') or None, api_base=api_base or None, max_tokens=apply_local_token_multiplier(200, llm_cfg), - debug=bool(datastore.data['settings']['application'].get('llm_debug', False)), + debug=get_llm_settings(datastore).debug, ) reply = text.strip() if not reply: @@ -233,7 +233,16 @@ def construct_llm_blueprint(datastore: ChangeDetectionStore): @login_optionally_required def llm_clear(): logger.debug("LLM configuration cleared by user") - datastore.data['settings']['application'].pop('llm', None) + # Strip only the credential / connection fields — user-set toggles, the + # global summary prompt, monthly budgets, and the system token counters + # all survive a "clear credentials" action. + llm = datastore.data['settings']['application'].get('llm') or {} + for key in ('model', 'api_key', 'api_base', 'provider_kind', 'local_token_multiplier'): + llm.pop(key, None) + if llm: + datastore.data['settings']['application']['llm'] = llm + else: + datastore.data['settings']['application'].pop('llm', None) datastore.commit() flash(gettext("AI / LLM configuration removed."), 'notice') return redirect(url_for('settings.settings_page') + '#ai') diff --git a/changedetectionio/blueprint/ui/diff.py b/changedetectionio/blueprint/ui/diff.py index fb0ccb27..7df5aed3 100644 --- a/changedetectionio/blueprint/ui/diff.py +++ b/changedetectionio/blueprint/ui/diff.py @@ -272,8 +272,10 @@ def construct_blueprint(datastore: ChangeDetectionStore): # Diff-pref flags + system prompt + active model are part of the cache key # so prompt or model changes bust the cache. - _max_summary_tokens = datastore.data['settings']['application'].get('llm_max_summary_tokens', 3000) - _llm_model = (datastore.data['settings']['application'].get('llm') or {}).get('model', '') + from changedetectionio.llm.evaluator import get_llm_settings + _ls = get_llm_settings(datastore) + _max_summary_tokens = _ls.max_summary_tokens + _llm_model = _ls.model cache_prompt = build_summary_cache_prompt( effective_prompt=get_effective_summary_prompt(watch, datastore), max_summary_tokens=_max_summary_tokens, diff --git a/changedetectionio/llm/evaluator.py b/changedetectionio/llm/evaluator.py index 085e9a2d..d8d9f24f 100644 --- a/changedetectionio/llm/evaluator.py +++ b/changedetectionio/llm/evaluator.py @@ -31,13 +31,30 @@ from .prompt_builder import ( ) from .response_parser import parse_eval_response, parse_preview_response, parse_setup_response -_DEFAULT_MAX_INPUT_CHARS = 100_000 +from changedetectionio.model.LLMSettings import ( + LLMSettings, + LLM_DEFAULT_MAX_INPUT_CHARS as _DEFAULT_MAX_INPUT_CHARS, + LLM_DEFAULT_MAX_SUMMARY_TOKENS, + LLM_DEFAULT_THINKING_BUDGET, +) def is_llm_features_disabled() -> bool: """True when the LLM_FEATURES_DISABLED env var is set to a truthy value.""" return bool(strtobool(os.getenv('LLM_FEATURES_DISABLED', ''))) + +def get_llm_settings(datastore) -> LLMSettings: + """Hydrate the LLM config dict at settings.application.llm into a validated model. + + Returns a default-constructed LLMSettings when the dict is missing or empty — + callers never have to None-check the result. The storage layer remains a plain + dict; this is only the validation/typing layer for reads. + """ + cfg = datastore.data.get('settings', {}).get('application', {}).get('llm') or {} + return LLMSettings.model_validate(cfg) + + def _get_max_input_chars(datastore) -> int: """Max input characters to send to the LLM. Resolution: env var → datastore → 100,000. Always returns at least 1 — unlimited is not permitted. @@ -45,10 +62,9 @@ def _get_max_input_chars(datastore) -> int: env_val = os.getenv('LLM_MAX_INPUT_CHARS', '').strip() if env_val.isdigit() and int(env_val) > 0: return int(env_val) - cfg = datastore.data.get('settings', {}).get('application', {}).get('llm') or {} - stored = cfg.get('max_input_chars') - if stored and int(stored) > 0: - return int(stored) + stored = get_llm_settings(datastore).max_input_chars + if stored and stored > 0: + return stored return _DEFAULT_MAX_INPUT_CHARS @@ -64,8 +80,6 @@ def _check_input_size(text: str, max_chars: int) -> None: ) -LLM_DEFAULT_THINKING_BUDGET = 0 # 0 = thinking disabled by default - def _thinking_extra_body(model: str, budget: int) -> dict | None: """Return litellm extra_body to control thinking for models that support it. For Gemini 2.5+: passes thinkingConfig with the given budget (0 = disabled). @@ -87,8 +101,6 @@ def _cached_system(text: str, model: str = '') -> dict: return {'role': 'system', 'content': text} -LLM_DEFAULT_MAX_SUMMARY_TOKENS = 3000 - # Output-token cap for the JSON-returning calls (intent eval, preview, setup/prefilter). # Mirrors client.py's _MAX_COMPLETION_TOKENS so the multiplier helper has a base value # to scale; cloud-LLM users hit this default unmodified, preserving prior cost defaults. @@ -254,9 +266,9 @@ def _runtime_llm_config(datastore) -> dict | None: the toggle is off. """ cfg = get_llm_config(datastore) - if not bool(datastore.data['settings']['application'].get('llm_enabled', True)): + if not get_llm_settings(datastore).enabled: if cfg: - logger.debug("LLM features disabled via settings (llm_enabled=False) — skipping LLM lookup") + logger.debug("LLM features disabled via settings (enabled=False) — skipping LLM lookup") return None return cfg @@ -423,6 +435,7 @@ def run_setup(watch, datastore, snapshot_text: str) -> None: url = watch.get('url', '') system_prompt = build_setup_system_prompt() user_prompt = build_setup_prompt(intent, snapshot_text, url=url) + settings = get_llm_settings(datastore) try: raw, tokens, *_ = llm_client.completion( @@ -434,8 +447,8 @@ def run_setup(watch, datastore, snapshot_text: str) -> None: api_key=cfg.get('api_key'), api_base=cfg.get('api_base'), max_tokens=apply_local_token_multiplier(JSON_RESPONSE_MAX_TOKENS, cfg), - extra_body=_thinking_extra_body(cfg['model'], int(datastore.data['settings']['application'].get('llm_thinking_budget', LLM_DEFAULT_THINKING_BUDGET) or 0)), - debug=bool(datastore.data['settings']['application'].get('llm_debug', False)), + extra_body=_thinking_extra_body(cfg['model'], settings.thinking_budget), + debug=settings.debug, ) _check_token_budget(watch, cfg, tokens) accumulate_global_tokens(datastore, tokens, model=cfg['model']) @@ -459,11 +472,7 @@ def get_effective_summary_prompt(watch, datastore) -> str: prompt, _ = resolve_llm_field(watch, datastore, 'llm_change_summary') if prompt: return prompt - global_default = ( - datastore.data.get('settings', {}) - .get('application', {}) - .get('llm_change_summary_default', '') or '' - ).strip() + global_default = get_llm_settings(datastore).change_summary_default.strip() return global_default or DEFAULT_CHANGE_SUMMARY_PROMPT @@ -573,8 +582,8 @@ def summarise_change(watch, datastore, diff: str, current_snapshot: str = '') -> title=title, ) - _thinking_budget = int(datastore.data['settings']['application'].get('llm_thinking_budget', LLM_DEFAULT_THINKING_BUDGET) or 0) - _extra_body = _thinking_extra_body(cfg['model'], _thinking_budget) + settings = get_llm_settings(datastore) + _extra_body = _thinking_extra_body(cfg['model'], settings.thinking_budget) try: _resp = llm_client.completion( @@ -586,14 +595,11 @@ def summarise_change(watch, datastore, diff: str, current_snapshot: str = '') -> api_key=cfg.get('api_key'), api_base=cfg.get('api_base'), max_tokens=apply_local_token_multiplier( - _summary_max_tokens( - diff, - max_cap=int(datastore.data['settings']['application'].get('llm_max_summary_tokens', LLM_DEFAULT_MAX_SUMMARY_TOKENS) or LLM_DEFAULT_MAX_SUMMARY_TOKENS), - ), + _summary_max_tokens(diff, max_cap=settings.max_summary_tokens), cfg, ), extra_body=_extra_body, - debug=bool(datastore.data['settings']['application'].get('llm_debug', False)), + debug=settings.debug, ) raw, tokens = _resp[0], _resp[1] input_tokens = _resp[2] if len(_resp) > 2 else 0 @@ -644,6 +650,7 @@ def preview_extract(watch, datastore, content: str) -> dict | None: system_prompt = build_preview_system_prompt() user_prompt = build_preview_prompt(intent, content, url=url, title=title) + settings = get_llm_settings(datastore) try: raw, tokens, *_ = llm_client.completion( @@ -655,8 +662,8 @@ def preview_extract(watch, datastore, content: str) -> dict | None: api_key=cfg.get('api_key'), api_base=cfg.get('api_base'), max_tokens=apply_local_token_multiplier(JSON_RESPONSE_MAX_TOKENS, cfg), - extra_body=_thinking_extra_body(cfg['model'], int(datastore.data['settings']['application'].get('llm_thinking_budget', LLM_DEFAULT_THINKING_BUDGET) or 0)), - debug=bool(datastore.data['settings']['application'].get('llm_debug', False)), + extra_body=_thinking_extra_body(cfg['model'], settings.thinking_budget), + debug=settings.debug, ) accumulate_global_tokens(datastore, tokens, model=cfg['model']) result = parse_preview_response(raw) @@ -730,6 +737,7 @@ def evaluate_change(watch, datastore, diff: str, current_snapshot: str = '') -> title=title, ) + settings = get_llm_settings(datastore) try: _resp = llm_client.completion( model=cfg['model'], @@ -740,8 +748,8 @@ def evaluate_change(watch, datastore, diff: str, current_snapshot: str = '') -> api_key=cfg.get('api_key'), api_base=cfg.get('api_base'), max_tokens=apply_local_token_multiplier(JSON_RESPONSE_MAX_TOKENS, cfg), - extra_body=_thinking_extra_body(cfg['model'], int(datastore.data['settings']['application'].get('llm_thinking_budget', LLM_DEFAULT_THINKING_BUDGET) or 0)), - debug=bool(datastore.data['settings']['application'].get('llm_debug', False)), + extra_body=_thinking_extra_body(cfg['model'], settings.thinking_budget), + debug=settings.debug, ) raw, tokens = _resp[0], _resp[1] input_tokens = _resp[2] if len(_resp) > 2 else 0 diff --git a/changedetectionio/model/App.py b/changedetectionio/model/App.py index 843e8244..c40f182c 100644 --- a/changedetectionio/model/App.py +++ b/changedetectionio/model/App.py @@ -2,7 +2,6 @@ from os import getenv from copy import deepcopy from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES, RSS_CONTENT_FORMAT_DEFAULT -from changedetectionio.llm.evaluator import LLM_DEFAULT_MAX_SUMMARY_TOKENS, LLM_DEFAULT_THINKING_BUDGET from changedetectionio.model.Tags import TagsDict from changedetectionio.notification import ( @@ -71,9 +70,9 @@ class model(dict): 'shared_diff_access': False, 'strip_ignored_lines': False, 'tags': None, # Initialized in __init__ with real datastore_path - 'llm_enabled': True, - 'llm_thinking_budget': LLM_DEFAULT_THINKING_BUDGET, - 'llm_max_summary_tokens': LLM_DEFAULT_MAX_SUMMARY_TOKENS, + # All LLM settings now live nested under application.llm.* (post-migration update_31). + # Defaults come from LLMSettings.model_validate({}) at read time — + # no need to pre-seed an empty {} here. 'webdriver_delay': None , # Extra delay in seconds before extracting text 'ui': { 'use_page_title_in_list': True, diff --git a/changedetectionio/model/LLMSettings.py b/changedetectionio/model/LLMSettings.py new file mode 100644 index 00000000..1088ee1b --- /dev/null +++ b/changedetectionio/model/LLMSettings.py @@ -0,0 +1,64 @@ +""" +LLMSettings — validation/typing layer over the LLM config dict. + +Storage shape (after migration update_31): everything lives under + datastore.data['settings']['application']['llm'] = { ... } + +Field names are stripped (enabled, debug, model, …). WTForms field names are +still llm_-prefixed (llm_enabled, llm_debug, …) and Pydantic Field aliases +bridge both sides, so callers don't repeat the rename. + +The store stays a plain dict (orjson-serialized) — this model is hydrated on +read (model_validate) and dumped on write (model_dump). Pydantic instances +are never held in datastore.data. +""" +from typing import ClassVar, Tuple + +from pydantic import BaseModel, ConfigDict, Field + + +LLM_DEFAULT_THINKING_BUDGET = 0 +LLM_DEFAULT_MAX_SUMMARY_TOKENS = 3000 +LLM_DEFAULT_LOCAL_TOKEN_MULTIPLIER = 5 +LLM_DEFAULT_MAX_INPUT_CHARS = 100_000 +LLM_DEFAULT_BUDGET_ACTION = 'skip_llm' + + +class LLMSettings(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + extra='allow', + ) + + enabled: bool = Field(default=True, alias='llm_enabled') + debug: bool = Field(default=False, alias='llm_debug') + override_diff_with_summary: bool = Field(default=True, alias='llm_override_diff_with_summary') + restock_use_fallback_extract: bool = Field(default=True, alias='llm_restock_use_fallback_extract') + thinking_budget: int = Field(default=LLM_DEFAULT_THINKING_BUDGET, alias='llm_thinking_budget') + max_summary_tokens: int = Field(default=LLM_DEFAULT_MAX_SUMMARY_TOKENS, alias='llm_max_summary_tokens') + budget_action: str = Field(default=LLM_DEFAULT_BUDGET_ACTION, alias='llm_budget_action') + change_summary_default: str = Field(default='', alias='llm_change_summary_default') + + model: str = Field(default='', alias='llm_model') + api_key: str = Field(default='', alias='llm_api_key') + api_base: str = Field(default='', alias='llm_api_base') + provider_kind: str = Field(default='', alias='llm_provider_kind') + local_token_multiplier: int = Field(default=LLM_DEFAULT_LOCAL_TOKEN_MULTIPLIER, alias='llm_local_token_multiplier') + token_budget_month: int = Field(default=0, alias='llm_token_budget_month') + max_input_chars: int = Field(default=LLM_DEFAULT_MAX_INPUT_CHARS, alias='llm_max_input_chars') + + tokens_total_cumulative: int = 0 + tokens_this_month: int = 0 + tokens_month_key: str = '' + cost_usd_total_cumulative: float = 0.0 + cost_usd_this_month: float = 0.0 + + # Runtime-managed counters that must survive form submissions. The settings + # POST handler strips these from form input before applying the merge. + PROTECTED_FIELDS: ClassVar[Tuple[str, ...]] = ( + 'tokens_total_cumulative', + 'tokens_this_month', + 'tokens_month_key', + 'cost_usd_total_cumulative', + 'cost_usd_this_month', + ) diff --git a/changedetectionio/notification/handler.py b/changedetectionio/notification/handler.py index d9d6b3d5..6329bdc5 100644 --- a/changedetectionio/notification/handler.py +++ b/changedetectionio/notification/handler.py @@ -376,7 +376,8 @@ def process_notification(n_object: NotificationContextData, datastore): # AI Change Summary: optionally replace {{ diff }} with the AI summary _llm_change_summary = (n_object.get('_llm_change_summary') or '').strip() - _override_diff = datastore.data['settings']['application'].get('llm_override_diff_with_summary', True) + from changedetectionio.llm.evaluator import get_llm_settings + _override_diff = get_llm_settings(datastore).override_diff_with_summary if _llm_change_summary and _override_diff: n_object['diff'] = _llm_change_summary diff --git a/changedetectionio/processors/restock_diff/plugins/llm_restock.py b/changedetectionio/processors/restock_diff/plugins/llm_restock.py index f8005e48..30ea8a27 100644 --- a/changedetectionio/processors/restock_diff/plugins/llm_restock.py +++ b/changedetectionio/processors/restock_diff/plugins/llm_restock.py @@ -196,19 +196,18 @@ def get_itemprop_availability_override(content, fetcher_name, fetcher_instance, logger.debug("LLM restock fallback: no datastore injected yet, skipping") return None - # Gate on the user setting (default True — enabled out of the box) - app_settings = datastore.data.get('settings', {}).get('application', {}) - if not app_settings.get('llm_restock_use_fallback_extract', True): - logger.debug("LLM restock fallback: disabled in settings") - return None - try: - from changedetectionio.llm.evaluator import _runtime_llm_config, accumulate_global_tokens + from changedetectionio.llm.evaluator import _runtime_llm_config, accumulate_global_tokens, get_llm_settings from changedetectionio.llm import client as llm_client except ImportError as e: logger.debug(f"LLM restock fallback: LLM libraries not available ({e})") return None + # Gate on the user setting (default True — enabled out of the box) + if not get_llm_settings(datastore).restock_use_fallback_extract: + logger.debug("LLM restock fallback: disabled in settings") + return None + # _runtime_llm_config returns None (with a debug log) when the master 'llm_enabled' # toggle is off, so this path is gated for free. llm_cfg = _runtime_llm_config(datastore) diff --git a/changedetectionio/processors/text_json_diff/difference.py b/changedetectionio/processors/text_json_diff/difference.py index c5e9ee98..625400ce 100644 --- a/changedetectionio/processors/text_json_diff/difference.py +++ b/changedetectionio/processors/text_json_diff/difference.py @@ -217,8 +217,10 @@ def render(watch, datastore, request, url_for, render_template, flash, redirect, llm_summary_prompt = _prompt # Must match the cache_prompt the worker writes and the UI ajax route reads — # using UI default diff prefs so the initial render finds the worker's pre-cache. - _max_summary_tokens = datastore.data['settings']['application'].get('llm_max_summary_tokens', 3000) - _llm_model = (datastore.data['settings']['application'].get('llm') or {}).get('model', '') + from changedetectionio.llm.evaluator import get_llm_settings + _ls = get_llm_settings(datastore) + _max_summary_tokens = _ls.max_summary_tokens + _llm_model = _ls.model _cache_prompt = build_summary_cache_prompt( effective_prompt=_prompt, max_summary_tokens=_max_summary_tokens, diff --git a/changedetectionio/store/updates.py b/changedetectionio/store/updates.py index 93637862..10c7de87 100644 --- a/changedetectionio/store/updates.py +++ b/changedetectionio/store/updates.py @@ -775,3 +775,45 @@ class DatastoreUpdatesMixin: tag.commit() logger.info(f"update_30: migrated tag {tag_uuid} restock_settings → processor_config_restock_diff") + def update_31(self): + """Fold flat application.llm_* settings into nested application.llm.* (stripped). + + Before: a handful of boolean toggles, the thinking budget, max summary tokens, + the budget action and the default summary prompt lived directly on + settings.application (llm_enabled, llm_thinking_budget, …). Every other LLM + field already lived under settings.application.llm.* with stripped names + (model, api_key, api_base, provider_kind, …). This unifies them so the new + LLMSettings pydantic model has a single home to read from / write to. + + Flat key wins on conflict — it was the most recently form-saved value, while + anything under llm.* was either set at creation time or is a system counter + which doesn't collide with the names we're moving in. + + Idempotent: skips when no flat keys are present. + """ + application = self.data['settings']['application'] + flat_to_nested = { + 'llm_enabled': 'enabled', + 'llm_debug': 'debug', + 'llm_thinking_budget': 'thinking_budget', + 'llm_max_summary_tokens': 'max_summary_tokens', + 'llm_change_summary_default': 'change_summary_default', + 'llm_override_diff_with_summary': 'override_diff_with_summary', + 'llm_restock_use_fallback_extract': 'restock_use_fallback_extract', + 'llm_budget_action': 'budget_action', + } + + present = [k for k in flat_to_nested if k in application] + if not present: + return + + nested = application.get('llm') or {} + for flat_key in present: + nested_key = flat_to_nested[flat_key] + nested[nested_key] = application[flat_key] + del application[flat_key] + + application['llm'] = nested + logger.info(f"update_31: folded {len(present)} flat llm_* keys into application.llm.* " + f"({', '.join(present)})") + diff --git a/changedetectionio/tests/llm/test_llm_restock_plugin.py b/changedetectionio/tests/llm/test_llm_restock_plugin.py index 2d41fffa..428df10c 100644 --- a/changedetectionio/tests/llm/test_llm_restock_plugin.py +++ b/changedetectionio/tests/llm/test_llm_restock_plugin.py @@ -14,8 +14,9 @@ def _make_datastore(llm_model='gpt-4o-mini', enabled=True): ds.data = { 'settings': { 'application': { - 'llm_restock_use_fallback_extract': enabled, 'llm': { + 'enabled': True, + 'restock_use_fallback_extract': enabled, 'model': llm_model, 'api_key': 'test-key', 'api_base': '', @@ -84,8 +85,8 @@ class TestLLMRestockPluginDisabled: ds.data = { 'settings': { 'application': { - 'llm_restock_use_fallback_extract': True, - # No 'llm' key → get_llm_config returns None + # No 'llm' key → get_llm_config returns None; + # restock_use_fallback_extract still defaults to True via LLMSettings } } } diff --git a/changedetectionio/tests/test_llm_change_summary.py b/changedetectionio/tests/test_llm_change_summary.py index 91030044..6d88f3d3 100644 --- a/changedetectionio/tests/test_llm_change_summary.py +++ b/changedetectionio/tests/test_llm_change_summary.py @@ -28,7 +28,11 @@ def _set_response(datastore_path, content): def _configure_llm(client): ds = client.application.config.get('DATASTORE') - ds.data['settings']['application']['llm'] = {'model': 'gpt-4o-mini', 'api_key': 'sk-test'} + # Merge into the existing llm dict so other test setup (e.g. change_summary_default + # set via _set_global_default) survives. + existing = ds.data['settings']['application'].get('llm') or {} + existing.update({'model': 'gpt-4o-mini', 'api_key': 'sk-test'}) + ds.data['settings']['application']['llm'] = existing # --------------------------------------------------------------------------- @@ -238,7 +242,9 @@ def test_llm_summary_ajax_error_displayed_not_silenced( # --------------------------------------------------------------------------- def _set_global_default(ds, prompt): - ds.data['settings']['application']['llm_change_summary_default'] = prompt + llm = ds.data['settings']['application'].get('llm') or {} + llm['change_summary_default'] = prompt + ds.data['settings']['application']['llm'] = llm def test_global_default_used_when_watch_and_tag_have_none( @@ -329,7 +335,7 @@ def test_hardcoded_fallback_when_nothing_set( watch['llm_change_summary'] = '' # Ensure global default is also empty - ds.data['settings']['application']['llm_change_summary_default'] = '' + _set_global_default(ds, '') assert get_effective_summary_prompt(watch, ds) == DEFAULT_CHANGE_SUMMARY_PROMPT @@ -391,8 +397,8 @@ def test_llm_summary_ajax_sets_last_viewed( def test_global_default_saved_and_loaded_via_settings_form( client, live_server, measure_memory_usage, datastore_path): """ - Submitting the settings form persists llm_change_summary_default at - settings.application level (not inside the llm credentials dict). + Submitting the settings form persists the global default prompt into + application.llm.change_summary_default (single nested home for all LLM settings). """ from changedetectionio.tests.util import live_server_setup live_server_setup(live_server) @@ -414,12 +420,11 @@ def test_global_default_saved_and_loaded_via_settings_form( assert b'Settings updated.' in res.data ds = client.application.config.get('DATASTORE') - stored = ds.data['settings']['application'].get('llm_change_summary_default', '') - assert stored == 'Saved global prompt.', f"Got: {stored!r}" - - # Must NOT be buried inside the llm credentials dict llm_dict = ds.data['settings']['application'].get('llm', {}) - assert 'change_summary_default' not in llm_dict + assert llm_dict.get('change_summary_default') == 'Saved global prompt.', f"Got: {llm_dict!r}" + + # And the old flat key must not be re-introduced + assert 'llm_change_summary_default' not in ds.data['settings']['application'] delete_all_watches(client) @@ -440,7 +445,11 @@ def test_global_default_survives_llm_clear( res = client.post(url_for('settings.llm.llm_clear'), follow_redirects=True) assert res.status_code == 200 - assert ds.data['settings']['application'].get('llm_change_summary_default') == 'Surviving prompt.' + llm_dict = ds.data['settings']['application'].get('llm') or {} + assert llm_dict.get('change_summary_default') == 'Surviving prompt.' + # The credential fields should be gone + assert 'model' not in llm_dict + assert 'api_key' not in llm_dict delete_all_watches(client) diff --git a/changedetectionio/worker.py b/changedetectionio/worker.py index e05f59b0..3813e347 100644 --- a/changedetectionio/worker.py +++ b/changedetectionio/worker.py @@ -436,9 +436,10 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec # Also gated on llm_enabled — a disabled LLM can't be spending tokens, # so the budget enforcement shouldn't suppress changes when the user # has explicitly switched LLM off. - from changedetectionio.llm.evaluator import is_llm_features_disabled as _is_llm_features_disabled - _llm_master_enabled = bool(datastore.data['settings']['application'].get('llm_enabled', True)) and not _is_llm_features_disabled() - _llm_budget_action = datastore.data['settings']['application'].get('llm_budget_action', 'skip_llm') + from changedetectionio.llm.evaluator import is_llm_features_disabled as _is_llm_features_disabled, get_llm_settings as _get_llm_settings + _llm_settings = _get_llm_settings(datastore) + _llm_master_enabled = _llm_settings.enabled and not _is_llm_features_disabled() + _llm_budget_action = _llm_settings.budget_action if _llm_master_enabled and _llm_budget_action == 'skip_check': from changedetectionio.llm.evaluator import is_global_token_budget_exceeded if is_global_token_budget_exceeded(datastore): @@ -548,8 +549,10 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec get_effective_summary_prompt, build_summary_cache_prompt, ) _llm_to_version = list(watch.history.keys())[-1] - _llm_max_summary_tokens = datastore.data['settings']['application'].get('llm_max_summary_tokens', 3000) - _llm_model = (datastore.data['settings']['application'].get('llm') or {}).get('model', '') + from changedetectionio.llm.evaluator import get_llm_settings as _get_llm_settings_inner + _ls = _get_llm_settings_inner(datastore) + _llm_max_summary_tokens = _ls.max_summary_tokens + _llm_model = _ls.model _llm_cache_prompt = build_summary_cache_prompt( effective_prompt=get_effective_summary_prompt(watch, datastore), max_summary_tokens=_llm_max_summary_tokens, diff --git a/requirements.txt b/requirements.txt index ef3cb452..5fa90d1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -148,6 +148,9 @@ pluggy ~= 1.6 # LLM intent-based change evaluation (multi-provider via litellm) litellm>=1.40.0,<1.83.1 # 1.83.1–1.83.14 exact-pin jsonschema==4.23.0, conflicting with openapi-spec-validator's >=4.24.0 floor; re-evaluate when litellm fixes this +# Used today for LLMSettings (model/LLMSettings.py); transitively pulled by litellm but pinned explicitly +# so the validation/typing layer doesn't disappear if litellm drops it. +pydantic>=2.0,<3.0 # BM25 relevance trimming for large snapshots (pure Python, no ML) rank-bm25>=0.2.2