Compare commits

..

1 Commits

51 changed files with 448 additions and 1402 deletions
+1 -1
View File
@@ -2,7 +2,7 @@
# Read more https://github.com/dgtlmoon/changedetection.io/wiki
# Semver means never use .01, or 00. Should be .1.
__version__ = '0.55.5'
__version__ = '0.55.4'
from changedetectionio.strtobool import strtobool
from json.decoder import JSONDecodeError
@@ -10,15 +10,12 @@ from flask_babel import gettext
from changedetectionio.store import ChangeDetectionStore
from changedetectionio.auth_decorator import login_optionally_required
from changedetectionio.model.LLMSettings import LLMSettings
def construct_blueprint(datastore: ChangeDetectionStore):
from changedetectionio.llm.evaluator import is_llm_features_disabled
from changedetectionio.blueprint.settings.llm import construct_llm_blueprint
settings_blueprint = Blueprint('settings', __name__, template_folder="templates")
if not is_llm_features_disabled():
from changedetectionio.blueprint.settings.llm import construct_llm_blueprint
settings_blueprint.register_blueprint(construct_llm_blueprint(datastore), url_prefix='/llm')
settings_blueprint.register_blueprint(construct_llm_blueprint(datastore), url_prefix='/llm')
@settings_blueprint.route("", methods=['GET', "POST"])
@login_optionally_required
@@ -33,12 +30,24 @@ def construct_blueprint(datastore: ChangeDetectionStore):
default = deepcopy(datastore.data['settings'])
# api_key is intentionally blanked on GET — PasswordField never re-renders
# its value, and a blank submission preserves the stored key.
default['llm'] = LLMSettings.model_validate(
datastore.data['settings']['application'].get('llm') or {}
).model_dump()
default['llm']['api_key'] = ''
# Pre-populate LLM sub-form fields from stored config (text fields only —
# PasswordField for api_key is intentionally left blank on GET).
_stored_llm = datastore.data['settings']['application'].get('llm') or {}
default['llm'] = {
'llm_model': _stored_llm.get('model', ''),
'llm_api_base': _stored_llm.get('api_base', ''),
'llm_provider_kind': _stored_llm.get('provider_kind', ''),
'llm_local_token_multiplier': _stored_llm.get('local_token_multiplier', 5),
'llm_change_summary_default': datastore.data['settings']['application'].get('llm_change_summary_default', ''),
'llm_override_diff_with_summary': datastore.data['settings']['application'].get('llm_override_diff_with_summary', True),
'llm_restock_use_fallback_extract': datastore.data['settings']['application'].get('llm_restock_use_fallback_extract', True),
'llm_debug': datastore.data['settings']['application'].get('llm_debug', False),
'llm_budget_action': datastore.data['settings']['application'].get('llm_budget_action', 'skip_llm'),
'llm_thinking_budget': str(datastore.data['settings']['application'].get('llm_thinking_budget', 0)),
'llm_max_summary_tokens': str(datastore.data['settings']['application'].get('llm_max_summary_tokens', 3000)),
'llm_token_budget_month': _stored_llm.get('token_budget_month', 0),
'llm_max_input_chars': _stored_llm.get('max_input_chars', 0),
}
if datastore.proxy_list is not None:
available_proxies = list(datastore.proxy_list.keys())
@@ -89,37 +98,79 @@ def construct_blueprint(datastore: ChangeDetectionStore):
datastore.data['settings']['application'].update(app_update)
# LLM config lives under settings.application.llm.* (post update_31).
# Hydrate the stored dict into LLMSettings, then merge form input over it.
# WTForms field names match LLMSettings field names exactly, so both sides
# of the merge use the same key shape.
existing_llm = LLMSettings.model_validate(
datastore.data['settings']['application'].get('llm') or {}
# Save LLM config separately under settings.application.llm.
# Token counters (tokens_total_cumulative, tokens_this_month, tokens_month_key)
# are system-managed and must never be overwritten by form submissions.
_LLM_PROTECTED_FIELDS = {
'tokens_total_cumulative', 'tokens_this_month', 'tokens_month_key',
'cost_usd_total_cumulative', 'cost_usd_this_month',
}
existing_llm = datastore.data['settings']['application'].get('llm') or {}
preserved_counters = {k: v for k, v in existing_llm.items() if k in _LLM_PROTECTED_FIELDS}
llm_data = form.data.get('llm') or {}
# PasswordField never re-populates its value on GET, so the submitted value
# is only non-empty when the user explicitly typed a new key.
# If blank, preserve the existing key so a settings save doesn't accidentally clear it.
submitted_api_key = (llm_data.get('llm_api_key') or '').strip()
effective_api_key = submitted_api_key if submitted_api_key else existing_llm.get('api_key', '')
# Application-level LLM settings (survive provider changes)
datastore.data['settings']['application']['llm_change_summary_default'] = (
llm_data.get('llm_change_summary_default') or ''
).strip()
datastore.data['settings']['application']['llm_override_diff_with_summary'] = (
bool(llm_data.get('llm_override_diff_with_summary', True))
)
datastore.data['settings']['application']['llm_restock_use_fallback_extract'] = (
bool(llm_data.get('llm_restock_use_fallback_extract', True))
)
datastore.data['settings']['application']['llm_debug'] = (
bool(llm_data.get('llm_debug', False))
)
datastore.data['settings']['application']['llm_budget_action'] = (
llm_data.get('llm_budget_action') or 'skip_llm'
)
datastore.data['settings']['application']['llm_thinking_budget'] = (
int(llm_data.get('llm_thinking_budget') or 0)
)
datastore.data['settings']['application']['llm_max_summary_tokens'] = (
int(llm_data.get('llm_max_summary_tokens') or 3000)
)
llm_form_input = dict(form.data.get('llm') or {})
# Monthly token budget — only save if env var is not set
import os as _os
if not _os.getenv('LLM_TOKEN_BUDGET_MONTH', '').strip():
_budget = llm_data.get('llm_token_budget_month') or 0
existing_llm['token_budget_month'] = int(_budget) if _budget else 0
# PasswordField never re-renders, so a blank submitted value means
# "keep stored key" — drop it from the merge.
if not (llm_form_input.get('api_key') or '').strip():
llm_form_input.pop('api_key', None)
# Max input chars — only save if env var is not set
if not _os.getenv('LLM_MAX_INPUT_CHARS', '').strip():
_max_chars = llm_data.get('llm_max_input_chars') or 0
existing_llm['max_input_chars'] = int(_max_chars) if _max_chars else 0
# Env-var overrides make these fields read-only in the UI — ignore form input.
if os.getenv('LLM_TOKEN_BUDGET_MONTH', '').strip():
llm_form_input.pop('token_budget_month', None)
if os.getenv('LLM_MAX_INPUT_CHARS', '').strip():
llm_form_input.pop('max_input_chars', None)
# System-managed counters must never come from the form.
for protected in LLMSettings.PROTECTED_FIELDS:
llm_form_input.pop(protected, None)
merged = LLMSettings.model_validate({**existing_llm.model_dump(), **llm_form_input})
# Clearing the model field strips only the provider-connection fields.
# User toggles, budgets, prompts and system counters survive (matches /llm/clear).
exclude = set(LLMSettings.CONNECTION_FIELDS) if not merged.model.strip() else None
datastore.data['settings']['application']['llm'] = merged.model_dump(exclude=exclude)
llm_config = {
'model': (llm_data.get('llm_model') or '').strip(),
'api_key': effective_api_key,
'api_base': (llm_data.get('llm_api_base') or '').strip(),
# Identifies a self-hosted OpenAI-compatible endpoint so reasoning-friendly
# token caps can be applied conditionally (cloud-LLM defaults stay tight).
'provider_kind': (llm_data.get('llm_provider_kind') or '').strip(),
'local_token_multiplier': int(llm_data.get('llm_local_token_multiplier') or 5),
'token_budget_month': existing_llm.get('token_budget_month', 0),
'max_input_chars': existing_llm.get('max_input_chars', 0),
**preserved_counters,
}
# Only store if a model is set
if llm_config['model']:
datastore.data['settings']['application']['llm'] = llm_config
else:
# Remove model config but retain counters for historical record
if preserved_counters:
datastore.data['settings']['application']['llm'] = preserved_counters
else:
datastore.data['settings']['application'].pop('llm', None)
# Handle dynamic worker count adjustment
old_worker_count = datastore.data['settings']['requests'].get('workers', 1)
+3 -12
View File
@@ -193,7 +193,7 @@ def construct_llm_blueprint(datastore: ChangeDetectionStore):
# via LLM_TIMEOUT). A shorter test-only timeout falsely fails on cold-starting
# cloud reasoning models (e.g. ollama.com hosting qwen3.5:397b takes ~60s on
# first hit) even though the same call succeeds in production.
from changedetectionio.llm.evaluator import apply_local_token_multiplier, get_llm_settings
from changedetectionio.llm.evaluator import apply_local_token_multiplier
text, total_tokens, input_tokens, output_tokens = completion(
model=model,
messages=[{'role': 'user', 'content':
@@ -201,7 +201,7 @@ def construct_llm_blueprint(datastore: ChangeDetectionStore):
api_key=llm_cfg.get('api_key') or None,
api_base=api_base or None,
max_tokens=apply_local_token_multiplier(200, llm_cfg),
debug=get_llm_settings(datastore).debug,
debug=bool(datastore.data['settings']['application'].get('llm_debug', False)),
)
reply = text.strip()
if not reply:
@@ -232,17 +232,8 @@ def construct_llm_blueprint(datastore: ChangeDetectionStore):
@llm_blueprint.route("/clear", methods=['POST'])
@login_optionally_required
def llm_clear():
from changedetectionio.model.LLMSettings import LLMSettings
logger.debug("LLM configuration cleared by user")
# Read existing config, write back a dict that omits the connection fields —
# so the saved dict no longer has model/api_key/api_base/etc.
# Toggles, prompts, budgets and counters survive.
settings = LLMSettings.model_validate(
datastore.data['settings']['application'].get('llm') or {}
)
datastore.data['settings']['application']['llm'] = settings.model_dump(
exclude=set(LLMSettings.CONNECTION_FIELDS)
)
datastore.data['settings']['application'].pop('llm', None)
datastore.commit()
flash(gettext("AI / LLM configuration removed."), 'notice')
return redirect(url_for('settings.settings_page') + '#ai')
@@ -34,9 +34,7 @@
<li class="tab"><a href="#plugin-{{ tab.plugin_id }}">{{ tab.tab_label }}</a></li>
{% endfor %}
{% endif %}
{% if not llm_features_disabled %}
<li class="tab"><a href="#ai">{{ _('AI / LLM') }}</a></li>
{% endif %}
<li class="tab"><a href="#info">{{ _('Info') }}</a></li>
</ul>
</div>
@@ -396,9 +394,7 @@ nav
</div>
{% endfor %}
{% endif %}
{% if not llm_features_disabled %}
{% include 'settings_llm_tab.html' %}
{% endif %}
<div class="tab-pane-inner" id="info">
<p><strong>{{ _('Uptime:') }}</strong> {{ uptime_seconds|format_duration }}</p>
<p><strong>{{ _('Python version:') }}</strong> {{ python_version }}</p>
@@ -69,17 +69,6 @@
{% call stab_pane('provider') %}
<p class="stab-section-title">{{ _('AI Provider') }}</p>
<div class="pure-control-group">
<label></label>
{{ form.llm.form.enabled() }}
<label for="{{ form.llm.form.enabled.id }}" style="display:inline; font-weight:normal;">
{{ form.llm.form.enabled.label.text }}
</label>
<span class="pure-form-message-inline">
{{ _('Master switch — when off, all AI lookups are skipped even if a provider is configured below.') }}
</span>
</div>
{% if not llm_env_configured and not (llm_config and llm_config.get('model')) %}
<div class="stab-overview-disclaimer">
<div class="stab-disclaimer-icon"></div>
@@ -125,22 +114,22 @@
</div>
<div class="pure-control-group">
{{ render_field(form.llm.form.api_key) }}
{{ render_field(form.llm.form.llm_api_key) }}
<span class="pure-form-message-inline" id="llm-key-hint"></span>
</div>
<div class="pure-control-group" id="llm-base-group" style="display:none">
{{ render_field(form.llm.form.api_base) }}
{{ render_field(form.llm.form.llm_api_base) }}
<span class="pure-form-message-inline">{{ _('Only needed for Ollama or custom/self-hosted endpoints. Leave blank for cloud providers.') }}</span>
</div>
{# Hidden field carrying the dropdown selection so the backend knows when to apply
reasoning-friendly token caps (Ollama and OpenAI-compatible endpoints, which commonly
serve reasoning models that need headroom for chain-of-thought to complete). #}
{{ form.llm.form.provider_kind() }}
{{ form.llm.form.llm_provider_kind() }}
<div class="pure-control-group" id="llm-local-advanced-group" style="display:none">
<label for="{{ form.llm.form.local_token_multiplier.id }}">{{ form.llm.form.local_token_multiplier.label.text }}</label>
{{ form.llm.form.local_token_multiplier() }}
<label for="{{ form.llm.form.llm_local_token_multiplier.id }}">{{ form.llm.form.llm_local_token_multiplier.label.text }}</label>
{{ form.llm.form.llm_local_token_multiplier() }}
<span class="pure-form-message-inline">
{{ _('Reasoning models (Qwen3, DeepSeek-R1, Gemma 3, etc.) emit chain-of-thought before the final answer. This multiplier scales every <code>max_tokens</code> cap for this endpoint to leave reasoning room. Defaults to %(default)s; raise it if responses come back truncated or empty, lower it (down to 1x) if you want tighter limits on a paid endpoint. Applied to Ollama and OpenAI-compatible endpoints — other cloud providers (OpenAI, Anthropic, Gemini) keep their original tight caps.', default='5x') | safe }}
</span>
@@ -163,7 +152,7 @@
</div>
<div class="pure-control-group">
{{ render_field(form.llm.form.model,
{{ render_field(form.llm.form.llm_model,
placeholder=_("Enter API key and click 'Load available models'")) }}
</div>
@@ -223,9 +212,9 @@
<div class="pure-control-group">
<label></label>
{{ form.llm.form.debug() }}
<label for="{{ form.llm.form.debug.id }}" style="display:inline; font-weight:normal;">
{{ form.llm.form.debug.label.text }}
{{ form.llm.form.llm_debug() }}
<label for="{{ form.llm.form.llm_debug.id }}" style="display:inline; font-weight:normal;">
{{ form.llm.form.llm_debug.label.text }}
</label>
<span class="pure-form-message-inline">
{{ _('Enables litellm verbose output (routed through loguru). Useful when diagnosing provider errors or empty responses. Leave off in production — generates a lot of log volume.') }}
@@ -243,10 +232,10 @@
<p class="stab-section-title">{{ _('Default AI Change Summary') }}</p>
<div class="pure-control-group">
{{ render_field(form.llm.form.change_summary_default) }}
{{ render_field(form.llm.form.llm_change_summary_default) }}
<span class="pure-form-message-inline">
{{ _('Used for all watches unless overridden by the watch or its tag/group.') }}
&nbsp;<a href="#" class="pure-button button-small" onclick="var t=document.getElementById('llm-change_summary_default'); if(!t.value && t.placeholder) t.value=t.placeholder; return false;">{{ _('Modify default prompt') }}</a>
&nbsp;<a href="#" class="pure-button button-small" onclick="var t=document.getElementById('llm-llm_change_summary_default'); if(!t.value && t.placeholder) t.value=t.placeholder; return false;">{{ _('Modify default prompt') }}</a>
</span>
</div>
@@ -259,9 +248,9 @@
{% if llm_config and llm_config.get('model') %}
<div class="pure-control-group">
<label></label>
{{ form.llm.form.override_diff_with_summary() }}
<label for="{{ form.llm.form.override_diff_with_summary.id }}" style="display:inline; font-weight:normal;">
{{ form.llm.form.override_diff_with_summary.label.text }}
{{ form.llm.form.llm_override_diff_with_summary() }}
<label for="{{ form.llm.form.llm_override_diff_with_summary.id }}" style="display:inline; font-weight:normal;">
{{ form.llm.form.llm_override_diff_with_summary.label.text }}
</label>
<span class="pure-form-message-inline">
{{ _('When enabled, the <code>%(diff)s</code> notification token shows the AI summary instead of the raw diff. Use <code>%(raw_diff)s</code> to always get the original.',
@@ -271,9 +260,9 @@
<div class="pure-control-group">
<label></label>
{{ form.llm.form.restock_use_fallback_extract() }}
<label for="{{ form.llm.form.restock_use_fallback_extract.id }}" style="display:inline; font-weight:normal;">
{{ form.llm.form.restock_use_fallback_extract.label.text }}
{{ form.llm.form.llm_restock_use_fallback_extract() }}
<label for="{{ form.llm.form.llm_restock_use_fallback_extract.id }}" style="display:inline; font-weight:normal;">
{{ form.llm.form.llm_restock_use_fallback_extract.label.text }}
</label>
<span class="pure-form-message-inline">
{{ _('When enabled, the AI will be used as a last resort to extract price and stock status from product pages where no structured metadata (JSON-LD, microdata, OpenGraph) is found.') }}
@@ -281,21 +270,21 @@
</div>
<div class="pure-control-group">
<label for="{{ form.llm.form.thinking_budget.id }}">{{ form.llm.form.thinking_budget.label.text }}</label>
{{ form.llm.form.thinking_budget() }}
<label for="{{ form.llm.form.llm_thinking_budget.id }}">{{ form.llm.form.llm_thinking_budget.label.text }}</label>
{{ form.llm.form.llm_thinking_budget() }}
<span class="pure-form-message-inline">{{ _('For Gemini 2.5+ models only. Thinking tokens improve reasoning quality but count against the output budget. Set to Off if summaries are being cut short.') }}</span>
</div>
<div class="pure-control-group">
<label for="{{ form.llm.form.max_summary_tokens.id }}">{{ form.llm.form.max_summary_tokens.label.text }}</label>
{{ form.llm.form.max_summary_tokens() }}
<label for="{{ form.llm.form.llm_max_summary_tokens.id }}">{{ form.llm.form.llm_max_summary_tokens.label.text }}</label>
{{ form.llm.form.llm_max_summary_tokens() }}
<span class="pure-form-message-inline">{{ _('Upper limit on tokens the AI may use when writing a change summary. Higher values allow longer summaries but cost more.') }}</span>
</div>
<div class="pure-control-group">
<label>{{ form.llm.form.budget_action.label.text }}</label>
<label>{{ form.llm.form.llm_budget_action.label.text }}</label>
<div>
{% for subfield in form.llm.form.budget_action %}
{% for subfield in form.llm.form.llm_budget_action %}
<label class="pure-radio" style="display:block; font-weight:normal; margin-bottom:0.3em;">
{{ subfield() }} {{ subfield.label.text }}
</label>
@@ -348,9 +337,9 @@
{% if llm_token_budget_month_env %}
<strong>{{ '{:,}'.format(llm_token_budget_month_env) }}</strong>
<span class="llm-env-badge">{{ _('(set via <code>LLM_TOKEN_BUDGET_MONTH</code>)') | safe }}</span>
<input type="hidden" name="llm-token_budget_month" value="{{ llm_token_budget_month_env }}">
<input type="hidden" name="llm-llm_token_budget_month" value="{{ llm_token_budget_month_env }}">
{% else %}
{{ form.llm.form.token_budget_month(placeholder=_('0 = unlimited'), value=llm_stored.get('token_budget_month', 0) or '') }}
{{ form.llm.form.llm_token_budget_month(placeholder=_('0 = unlimited'), value=llm_stored.get('token_budget_month', 0) or '') }}
<span class="llm-field-hint">{{ _('tokens (0 = unlimited)') }}</span>
{% endif %}
</span>
@@ -365,21 +354,14 @@
<span class="llm-usage-row-label">{{ _('Max input characters') }}</span>
<span class="llm-usage-row-value">
{% if llm_max_input_chars_env %}
{{ form.llm.form.max_input_chars(value=llm_max_input_chars_env, readonly=True, style="width:10em;opacity:0.6;cursor:not-allowed;") }}
{{ form.llm.form.llm_max_input_chars(value=llm_max_input_chars_env, readonly=True, style="width:10em;opacity:0.6;cursor:not-allowed;") }}
<span class="llm-env-badge">{{ _('(set via <code>LLM_MAX_INPUT_CHARS</code>)') | safe }}</span>
{% else %}
{{ form.llm.form.max_input_chars(placeholder='100000', value=llm_stored.get('max_input_chars', 100000) or '') }}
{{ form.llm.form.llm_max_input_chars(placeholder='100000', value=llm_stored.get('max_input_chars', 100000) or '') }}
<span class="llm-field-hint">{{ _('characters — currently enforcing: %(limit)s', limit='{:,}'.format(llm_effective_max_input_chars)) }}</span>
{% endif %}
</span>
</div>
<div class="llm-usage-row">
<span class="llm-usage-row-label">{{ _('Max tokens per watch per period') }}</span>
<span class="llm-usage-row-value">
{{ form.llm.form.max_tokens_per_count_period(placeholder=_('0 = unlimited'), value=llm_stored.get('max_tokens_per_count_period', 0) or '') }}
<span class="llm-field-hint">{{ _('tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = unlimited)') }}</span>
</span>
</div>
</div>
{% else %}
@@ -392,9 +374,9 @@
{% if llm_token_budget_month_env %}
<strong>{{ '{:,}'.format(llm_token_budget_month_env) }}</strong>
<span class="llm-env-badge">{{ _('(set via <code>LLM_TOKEN_BUDGET_MONTH</code>)') | safe }}</span>
<input type="hidden" name="llm-token_budget_month" value="{{ llm_token_budget_month_env }}">
<input type="hidden" name="llm-llm_token_budget_month" value="{{ llm_token_budget_month_env }}">
{% else %}
{{ form.llm.form.token_budget_month(placeholder=_('0 = unlimited'), value=llm_stored.get('token_budget_month', 0) or '') }}
{{ form.llm.form.llm_token_budget_month(placeholder=_('0 = unlimited'), value=llm_stored.get('token_budget_month', 0) or '') }}
<span class="llm-field-hint">{{ _('tokens per month (0 = unlimited)') }}</span>
{% endif %}
</span>
@@ -403,21 +385,14 @@
<span class="llm-usage-row-label">{{ _('Max input characters') }}</span>
<span class="llm-usage-row-value">
{% if llm_max_input_chars_env %}
{{ form.llm.form.max_input_chars(value=llm_max_input_chars_env, readonly=True, style="width:10em;opacity:0.6;cursor:not-allowed;") }}
{{ form.llm.form.llm_max_input_chars(value=llm_max_input_chars_env, readonly=True, style="width:10em;opacity:0.6;cursor:not-allowed;") }}
<span class="llm-env-badge">{{ _('(set via <code>LLM_MAX_INPUT_CHARS</code>)') | safe }}</span>
{% else %}
{{ form.llm.form.max_input_chars(placeholder='100000', value=llm_stored.get('max_input_chars', 100000) or '') }}
{{ form.llm.form.llm_max_input_chars(placeholder='100000', value=llm_stored.get('max_input_chars', 100000) or '') }}
<span class="llm-field-hint">{{ _('characters — currently enforcing: %(limit)s', limit='{:,}'.format(llm_effective_max_input_chars)) }}</span>
{% endif %}
</span>
</div>
<div class="llm-usage-row">
<span class="llm-usage-row-label">{{ _('Max tokens per watch per period') }}</span>
<span class="llm-usage-row-value">
{{ form.llm.form.max_tokens_per_count_period(placeholder=_('0 = unlimited'), value=llm_stored.get('max_tokens_per_count_period', 0) or '') }}
<span class="llm-field-hint">{{ _('tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = unlimited)') }}</span>
</span>
</div>
</div>
{% endif %}
{% endcall %}
@@ -448,8 +423,8 @@
const baseGroup = document.getElementById('llm-base-group');
const modelSelGrp = document.getElementById('llm-model-select-group');
const localAdvGrp = document.getElementById('llm-local-advanced-group');
const baseField = document.querySelector('[name="llm-api_base"]');
const kindField = document.querySelector('[name="llm-provider_kind"]');
const baseField = document.querySelector('[name="llm-llm_api_base"]');
const kindField = document.querySelector('[name="llm-llm_provider_kind"]');
const hint = document.getElementById('llm-key-hint');
fetchGroup.style.display = LIVE_PROVIDERS.includes(provider) ? '' : 'none';
@@ -477,8 +452,8 @@
window.llmFetchModels = async function () {
const provider = document.getElementById('llm-provider').value;
const apiKey = document.querySelector('[name="llm-api_key"]').value.trim();
const apiBase = document.querySelector('[name="llm-api_base"]').value.trim();
const apiKey = document.querySelector('[name="llm-llm_api_key"]').value.trim();
const apiBase = document.querySelector('[name="llm-llm_api_base"]').value.trim();
const btn = document.getElementById('llm-fetch-btn');
const statusEl = document.getElementById('llm-fetch-status');
const selGroup = document.getElementById('llm-model-select-group');
@@ -513,7 +488,7 @@
}
modelSel.innerHTML = '<option value="">{{ _("— choose a model —") }}</option>';
const currentModel = document.querySelector('[name="llm-model"]').value.trim();
const currentModel = document.querySelector('[name="llm-llm_model"]').value.trim();
for (const m of data.models) {
const opt = document.createElement('option');
opt.value = m;
@@ -535,7 +510,7 @@
};
window.llmOnModelPick = function (value) {
if (value) document.querySelector('[name="llm-model"]').value = value;
if (value) document.querySelector('[name="llm-llm_model"]').value = value;
};
window.llmRunTest = async function () {
@@ -551,11 +526,11 @@
// testing a config change. Endpoint falls back to the stored datastore values
// for any field we don't send.
const params = new URLSearchParams();
const model = (document.querySelector('[name="llm-model"]') || {}).value || '';
const apiKey = (document.querySelector('[name="llm-api_key"]') || {}).value || '';
const apiBase = (document.querySelector('[name="llm-api_base"]') || {}).value || '';
const kind = (document.querySelector('[name="llm-provider_kind"]') || {}).value || '';
const mult = (document.querySelector('[name="llm-local_token_multiplier"]') || {}).value || '';
const model = (document.querySelector('[name="llm-llm_model"]') || {}).value || '';
const apiKey = (document.querySelector('[name="llm-llm_api_key"]') || {}).value || '';
const apiBase = (document.querySelector('[name="llm-llm_api_base"]') || {}).value || '';
const kind = (document.querySelector('[name="llm-llm_provider_kind"]') || {}).value || '';
const mult = (document.querySelector('[name="llm-llm_local_token_multiplier"]') || {}).value || '';
if (model.trim()) params.set('model', model.trim());
if (apiKey.trim()) params.set('api_key', apiKey.trim());
if (apiBase.trim()) params.set('api_base', apiBase.trim());
@@ -585,7 +560,7 @@
// On page load: detect and pre-select provider from current model
(function detectCurrentProvider() {
const modelField = document.querySelector('[name="llm-model"]');
const modelField = document.querySelector('[name="llm-llm_model"]');
if (!modelField) return;
const m = modelField.value.trim();
if (!m) return;
@@ -596,7 +571,7 @@
else if (m.startsWith('openrouter/')) guessed = 'openrouter';
else if (m.startsWith('openai/')) {
// openai/<model> + custom api_base = self-hosted OpenAI-compatible (vLLM etc.)
const baseField = document.querySelector('[name="llm-api_base"]');
const baseField = document.querySelector('[name="llm-llm_api_base"]');
guessed = (baseField && baseField.value.trim()) ? 'openai_compatible' : 'openai';
}
else if (m.startsWith('claude')) guessed = 'anthropic';
+2 -4
View File
@@ -272,10 +272,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
# Diff-pref flags + system prompt + active model are part of the cache key
# so prompt or model changes bust the cache.
from changedetectionio.llm.evaluator import get_llm_settings
_ls = get_llm_settings(datastore)
_max_summary_tokens = _ls.max_summary_tokens
_llm_model = _ls.model
_max_summary_tokens = datastore.data['settings']['application'].get('llm_max_summary_tokens', 3000)
_llm_model = (datastore.data['settings']['application'].get('llm') or {}).get('model', '')
cache_prompt = build_summary_cache_prompt(
effective_prompt=get_effective_summary_prompt(watch, datastore),
max_summary_tokens=_max_summary_tokens,
@@ -57,9 +57,7 @@
{% if capabilities.supports_visual_selector %}
<li class="tab"><a id="visualselector-tab" href="#visualselector">{{ _('Visual Filter Selector') }}</a></li>
{% endif %}
{% if not llm_features_disabled %}
<li class="tab"><a href="#ai-llm">{{ _('AI / LLM') }}</a></li>
{% endif %}
{% if capabilities.supports_text_filters_and_triggers %}
<li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">{{ _('Filters & Triggers') }}</a></li>
<li class="tab" id="conditions-tab"><a href="#conditions">{{ _('Conditions') }}</a></li>
@@ -323,11 +321,9 @@ Math: {{ 1 + 1 }}") }}
</div>
</div>
</div>
{% if not llm_features_disabled %}
<div class="tab-pane-inner" id="ai-llm">
{% include "edit/include_llm_intent.html" %}
</div>
{% endif %}
<div class="tab-pane-inner" id="filters-and-triggers">
<span id="activate-text-preview" class="pure-button pure-button-primary button-xsmall">{{ _('Activate preview') }}</span>
@@ -507,7 +503,7 @@ Math: {{ 1 + 1 }}") }}
<td>{{ _('Server type reply') }}</td>
<td>{{ watch.get('remote_server_reply') }}</td>
</tr>
{% if not llm_features_disabled and settings_application.get('llm', {}).get('model') %}
{% if settings_application.get('llm', {}).get('model') %}
<tr>
<td>{{ _('AI tokens (last check)') }}</td>
<td>{{ "{:,}".format(watch.get('llm_last_tokens_used') or 0) }}</td>
-5
View File
@@ -522,11 +522,6 @@ def changedetection_app(config=None, datastore_o=None):
available_languages=available_languages
)
@app.context_processor
def inject_llm_features_disabled():
from changedetectionio.llm.evaluator import is_llm_features_disabled
return dict(llm_features_disabled=is_llm_features_disabled())
# Set up a request hook to check authentication for all routes
@app.before_request
def check_authentication():
+25 -24
View File
@@ -1108,12 +1108,12 @@ class globalSettingsLLMForm(Form):
gemini/gemini-2.0-flash Google Gemini
azure/gpt-4o Azure OpenAI
"""
model = StringField(
llm_model = StringField(
_l('Model'),
validators=[validators.Optional()],
render_kw={"placeholder": "gpt-4o-mini", "style": "width: 24em;"},
)
api_key = PasswordField(
llm_api_key = PasswordField(
_l('API Key'),
validators=[validators.Optional()],
render_kw={
@@ -1121,7 +1121,7 @@ class globalSettingsLLMForm(Form):
"style": "width: 24em;",
},
)
api_base = StringField(
llm_api_base = StringField(
_l('API Base URL'),
validators=[validators.Optional(), validateLLMApiBaseSafe()],
render_kw={
@@ -1132,7 +1132,7 @@ class globalSettingsLLMForm(Form):
# Persisted by the Provider dropdown JS — lets the backend distinguish a self-hosted
# OpenAI-compatible endpoint (vLLM, LM Studio, llama.cpp) from cloud OpenAI, so we can
# apply reasoning-friendly token caps only when the user opted in.
provider_kind = HiddenField(
llm_provider_kind = HiddenField(
validators=[validators.Optional()],
default='',
)
@@ -1144,13 +1144,13 @@ class globalSettingsLLMForm(Form):
# OpenRouter) stay on the original tight caps so existing users see no
# behavior or cost change. Users on paid Ollama / openai_compatible endpoints
# who care about cost can dial this down to 1x.
local_token_multiplier = IntegerField(
llm_local_token_multiplier = IntegerField(
_l('Token multiplier for local reasoning models'),
validators=[validators.Optional(), validators.NumberRange(min=1, max=20)],
default=5,
render_kw={"placeholder": "5", "style": "width: 6em;"},
)
change_summary_default = TextAreaField(
llm_change_summary_default = TextAreaField(
_l('Default AI Change Summary prompt'),
validators=[validators.Optional(), validators.Length(max=2000)],
render_kw={
@@ -1160,8 +1160,8 @@ class globalSettingsLLMForm(Form):
},
default='',
)
max_tokens_per_count_period = IntegerField(
_l('Max tokens per watch per period'),
llm_max_tokens_per_check = IntegerField(
_l('Max tokens per check'),
validators=[validators.Optional(), validators.NumberRange(min=0)],
default=0,
render_kw={
@@ -1169,13 +1169,22 @@ class globalSettingsLLMForm(Form):
"style": "width: 8em;",
},
)
token_budget_month = IntegerField(
llm_max_tokens_cumulative = IntegerField(
_l('Max cumulative tokens (per watch)'),
validators=[validators.Optional(), validators.NumberRange(min=0)],
default=0,
render_kw={
"placeholder": "0 = unlimited",
"style": "width: 8em;",
},
)
llm_token_budget_month = IntegerField(
_l('Monthly token budget'),
validators=[validators.Optional(), validators.NumberRange(min=0)],
default=0,
render_kw={"style": "width: 10em;"},
)
max_input_chars = IntegerField(
llm_max_input_chars = IntegerField(
_l('Max input characters'),
validators=[validators.Optional(), validators.NumberRange(min=1)],
default=100000,
@@ -1184,27 +1193,19 @@ class globalSettingsLLMForm(Form):
"style": "width: 10em;",
},
)
# Master on/off switch for ALL LLM lookups at runtime. When False, every entry point
# in evaluator.py (and the restock fallback) short-circuits with a logger.debug
# message — even if a provider+model is still configured. Saved config and the
# "configured" badge remain visible so the user can toggle back on without re-entering.
enabled = BooleanField(
_l('Enable AI / LLM features'),
default=True,
)
override_diff_with_summary = BooleanField(
llm_override_diff_with_summary = BooleanField(
_l('Replace {{diff}} notification token with AI summary'),
default=True,
)
restock_use_fallback_extract = BooleanField(
llm_restock_use_fallback_extract = BooleanField(
_l('Use LLM as a fallback for extracting price and restock info'),
default=True,
)
debug = BooleanField(
llm_debug = BooleanField(
_l('Enable LLM debug logging'),
default=False,
)
thinking_budget = SelectField(
llm_thinking_budget = SelectField(
_l('AI thinking budget (tokens)'),
choices=[
('0', _l('Off (no thinking)')),
@@ -1215,7 +1216,7 @@ class globalSettingsLLMForm(Form):
default=str(LLM_DEFAULT_THINKING_BUDGET),
validators=[validators.Optional()],
)
max_summary_tokens = SelectField(
llm_max_summary_tokens = SelectField(
_l('Max AI summary length (tokens)'),
choices=[
('500', '500'),
@@ -1228,7 +1229,7 @@ class globalSettingsLLMForm(Form):
default=str(LLM_DEFAULT_MAX_SUMMARY_TOKENS),
validators=[validators.Optional()],
)
budget_action = RadioField(
llm_budget_action = RadioField(
_l('When monthly token budget is reached'),
choices=[
('skip_llm', _l('Skip AI summarisation only (watch still checks)')),
+68 -119
View File
@@ -20,8 +20,6 @@ from dataclasses import dataclass
from datetime import datetime, timezone
from loguru import logger
from changedetectionio.strtobool import strtobool
from . import client as llm_client
from .prompt_builder import (
build_change_summary_prompt, build_change_summary_system_prompt,
@@ -31,29 +29,7 @@ from .prompt_builder import (
)
from .response_parser import parse_eval_response, parse_preview_response, parse_setup_response
from changedetectionio.model.LLMSettings import (
LLMSettings,
LLM_DEFAULT_MAX_INPUT_CHARS as _DEFAULT_MAX_INPUT_CHARS,
LLM_DEFAULT_MAX_SUMMARY_TOKENS,
LLM_DEFAULT_THINKING_BUDGET,
)
def is_llm_features_disabled() -> bool:
"""True when the LLM_FEATURES_DISABLED env var is set to a truthy value."""
return bool(strtobool(os.getenv('LLM_FEATURES_DISABLED', '')))
def get_llm_settings(datastore) -> LLMSettings:
"""Hydrate the LLM config dict at settings.application.llm into a validated model.
Returns a default-constructed LLMSettings when the dict is missing or empty
callers never have to None-check the result. The storage layer remains a plain
dict; this is only the validation/typing layer for reads.
"""
cfg = datastore.data.get('settings', {}).get('application', {}).get('llm') or {}
return LLMSettings.model_validate(cfg)
_DEFAULT_MAX_INPUT_CHARS = 100_000
def _get_max_input_chars(datastore) -> int:
"""Max input characters to send to the LLM. Resolution: env var → datastore → 100,000.
@@ -62,9 +38,10 @@ def _get_max_input_chars(datastore) -> int:
env_val = os.getenv('LLM_MAX_INPUT_CHARS', '').strip()
if env_val.isdigit() and int(env_val) > 0:
return int(env_val)
stored = get_llm_settings(datastore).max_input_chars
if stored and stored > 0:
return stored
cfg = datastore.data.get('settings', {}).get('application', {}).get('llm') or {}
stored = cfg.get('max_input_chars')
if stored and int(stored) > 0:
return int(stored)
return _DEFAULT_MAX_INPUT_CHARS
@@ -80,6 +57,8 @@ def _check_input_size(text: str, max_chars: int) -> None:
)
LLM_DEFAULT_THINKING_BUDGET = 0 # 0 = thinking disabled by default
def _thinking_extra_body(model: str, budget: int) -> dict | None:
"""Return litellm extra_body to control thinking for models that support it.
For Gemini 2.5+: passes thinkingConfig with the given budget (0 = disabled).
@@ -101,6 +80,8 @@ def _cached_system(text: str, model: str = '') -> dict:
return {'role': 'system', 'content': text}
LLM_DEFAULT_MAX_SUMMARY_TOKENS = 3000
# Output-token cap for the JSON-returning calls (intent eval, preview, setup/prefilter).
# Mirrors client.py's _MAX_COMPLETION_TOKENS so the multiplier helper has a base value
# to scale; cloud-LLM users hit this default unmodified, preserving prior cost defaults.
@@ -226,8 +207,6 @@ def get_llm_config(datastore) -> dict | None:
1. Environment variables: LLM_MODEL, LLM_API_KEY, LLM_API_BASE
2. Datastore settings (set via UI)
"""
if is_llm_features_disabled():
return None
# 1. Environment variable override
env_model = os.getenv('LLM_MODEL', '').strip()
if env_model:
@@ -246,33 +225,9 @@ def get_llm_config(datastore) -> dict | None:
def llm_configured_via_env() -> bool:
"""True when LLM config comes from environment variables, not the UI."""
if is_llm_features_disabled():
return False
return bool(os.getenv('LLM_MODEL', '').strip())
def _runtime_llm_config(datastore) -> dict | None:
"""
Runtime gate used by every LLM entry point in this module (and the restock
fallback). Returns the resolved config dict only when both:
- the master 'llm_enabled' toggle is on (default True)
- a provider+model is actually configured
When the toggle is off but a config exists, logs a debug message and returns
None so callers fall through their existing "not configured" early-return path.
The settings UI deliberately still calls get_llm_config() directly so the
"AI / LLM configured: ..." badge keeps showing the saved provider even while
the toggle is off.
"""
cfg = get_llm_config(datastore)
if not get_llm_settings(datastore).enabled:
if cfg:
logger.debug("LLM features disabled via settings (enabled=False) — skipping LLM lookup")
return None
return cfg
# ---------------------------------------------------------------------------
# Global monthly token budget
# ---------------------------------------------------------------------------
@@ -343,22 +298,25 @@ def accumulate_global_tokens(datastore, tokens: int,
current_month = _get_month_key()
cost = _estimate_cost_usd(model, input_tokens, output_tokens)
settings = get_llm_settings(datastore)
# Work on the live dict in-place (or create a stub if llm key is absent)
app_settings = datastore.data['settings']['application']
if 'llm' not in app_settings:
app_settings['llm'] = {}
llm_cfg = app_settings['llm']
# Month rollover: reset monthly counters
if settings.tokens_month_key != current_month:
settings.tokens_this_month = 0
settings.cost_usd_this_month = 0.0
settings.tokens_month_key = current_month
if llm_cfg.get('tokens_month_key') != current_month:
llm_cfg['tokens_this_month'] = 0
llm_cfg['cost_usd_this_month'] = 0.0
llm_cfg['tokens_month_key'] = current_month
settings.tokens_total_cumulative += tokens
settings.tokens_this_month += tokens
settings.cost_usd_total_cumulative += cost
settings.cost_usd_this_month += cost
llm_cfg['tokens_total_cumulative'] = (llm_cfg.get('tokens_total_cumulative') or 0) + tokens
llm_cfg['tokens_this_month'] = (llm_cfg.get('tokens_this_month') or 0) + tokens
llm_cfg['cost_usd_total_cumulative'] = (llm_cfg.get('cost_usd_total_cumulative') or 0.0) + cost
llm_cfg['cost_usd_this_month'] = (llm_cfg.get('cost_usd_this_month') or 0.0) + cost
# Round-trip through model_dump so storage stays a plain dict and the schema
# contract (extra='forbid', type coercion) is re-enforced on every write.
datastore.data['settings']['application']['llm'] = settings.model_dump()
# Persist immediately — token accounting must survive restarts
datastore.commit()
@@ -386,44 +344,31 @@ def is_global_token_budget_exceeded(datastore) -> bool:
def _check_token_budget(watch, cfg, tokens_this_call: int = 0) -> bool:
"""
Per-watch per-period token cap.
Period is currently month (matches the global counter rollover); the field
name `max_tokens_per_count_period` is period-agnostic so a configurable
day/week/month can land later without renaming storage.
On non-zero tokens_this_call:
- rolls over watch['llm_tokens_this_period'] if a new period started
- increments the per-period counter
- also increments the existing lifetime counter (UI stat, unchanged)
Returns False once the per-period counter exceeds max_tokens_per_count_period
so subsequent evaluate_change calls bail out for this watch until rollover.
Note: only evaluate_change actually gates on the return value (the other
callers invoke this for the side-effect of accumulating tokens).
Check token budget limits. Returns True if within budget, False if exceeded.
Also accumulates tokens_this_call into watch['llm_tokens_used_cumulative'].
"""
if tokens_this_call > 0:
current_period = _get_month_key()
# Rollover: new period zeroes the per-period counter
if watch.get('llm_tokens_period_key') != current_period:
watch['llm_tokens_this_period'] = 0
watch['llm_tokens_period_key'] = current_period
watch['llm_tokens_this_period'] = (watch.get('llm_tokens_this_period') or 0) + tokens_this_call
# Informational lifetime counter (UI shows this; not used for the cap)
watch['llm_tokens_used_cumulative'] = (watch.get('llm_tokens_used_cumulative') or 0) + tokens_this_call
current = watch.get('llm_tokens_used_cumulative') or 0
watch['llm_tokens_used_cumulative'] = current + tokens_this_call
max_per_period = int(cfg.get('max_tokens_per_count_period') or 0)
if max_per_period:
# Pre-flight (tokens_this_call=0) and post-call paths both read the
# same counter — but a stale period key means "no usage yet this period".
if watch.get('llm_tokens_period_key') == _get_month_key():
total = watch.get('llm_tokens_this_period') or 0
if total > max_per_period:
logger.warning(
f"LLM per-period token budget exceeded for {watch.get('uuid')}: "
f"{total} tokens > limit {max_per_period}"
)
return False
max_per_check = int(cfg.get('max_tokens_per_check') or 0)
max_cumulative = int(cfg.get('max_tokens_cumulative') or 0)
if max_per_check and tokens_this_call > max_per_check:
logger.warning(
f"LLM token budget exceeded for {watch.get('uuid')}: "
f"{tokens_this_call} tokens > per-check limit {max_per_check}"
)
return False
if max_cumulative:
total = watch.get('llm_tokens_used_cumulative') or 0
if total > max_cumulative:
logger.warning(
f"LLM cumulative token budget exceeded for {watch.get('uuid')}: "
f"{total} tokens > limit {max_cumulative}"
)
return False
return True
@@ -434,7 +379,7 @@ def run_setup(watch, datastore, snapshot_text: str) -> None:
Stores result in watch['llm_prefilter'] (str selector or None).
Called once when intent is first set, and again if pre-filter returns zero matches.
"""
cfg = _runtime_llm_config(datastore)
cfg = get_llm_config(datastore)
if not cfg:
return
@@ -445,7 +390,6 @@ def run_setup(watch, datastore, snapshot_text: str) -> None:
url = watch.get('url', '')
system_prompt = build_setup_system_prompt()
user_prompt = build_setup_prompt(intent, snapshot_text, url=url)
settings = get_llm_settings(datastore)
try:
raw, tokens, *_ = llm_client.completion(
@@ -457,8 +401,8 @@ def run_setup(watch, datastore, snapshot_text: str) -> None:
api_key=cfg.get('api_key'),
api_base=cfg.get('api_base'),
max_tokens=apply_local_token_multiplier(JSON_RESPONSE_MAX_TOKENS, cfg),
extra_body=_thinking_extra_body(cfg['model'], settings.thinking_budget),
debug=settings.debug,
extra_body=_thinking_extra_body(cfg['model'], int(datastore.data['settings']['application'].get('llm_thinking_budget', LLM_DEFAULT_THINKING_BUDGET) or 0)),
debug=bool(datastore.data['settings']['application'].get('llm_debug', False)),
)
_check_token_budget(watch, cfg, tokens)
accumulate_global_tokens(datastore, tokens, model=cfg['model'])
@@ -482,7 +426,11 @@ def get_effective_summary_prompt(watch, datastore) -> str:
prompt, _ = resolve_llm_field(watch, datastore, 'llm_change_summary')
if prompt:
return prompt
global_default = get_llm_settings(datastore).change_summary_default.strip()
global_default = (
datastore.data.get('settings', {})
.get('application', {})
.get('llm_change_summary_default', '') or ''
).strip()
return global_default or DEFAULT_CHANGE_SUMMARY_PROMPT
@@ -561,7 +509,7 @@ def summarise_change(watch, datastore, diff: str, current_snapshot: str = '') ->
The result replaces {{ diff }} in notifications so the user gets a
readable description instead of raw +/- diff lines.
"""
cfg = _runtime_llm_config(datastore)
cfg = get_llm_config(datastore)
if not cfg:
return ''
@@ -592,8 +540,8 @@ def summarise_change(watch, datastore, diff: str, current_snapshot: str = '') ->
title=title,
)
settings = get_llm_settings(datastore)
_extra_body = _thinking_extra_body(cfg['model'], settings.thinking_budget)
_thinking_budget = int(datastore.data['settings']['application'].get('llm_thinking_budget', LLM_DEFAULT_THINKING_BUDGET) or 0)
_extra_body = _thinking_extra_body(cfg['model'], _thinking_budget)
try:
_resp = llm_client.completion(
@@ -605,11 +553,14 @@ def summarise_change(watch, datastore, diff: str, current_snapshot: str = '') ->
api_key=cfg.get('api_key'),
api_base=cfg.get('api_base'),
max_tokens=apply_local_token_multiplier(
_summary_max_tokens(diff, max_cap=settings.max_summary_tokens),
_summary_max_tokens(
diff,
max_cap=int(datastore.data['settings']['application'].get('llm_max_summary_tokens', LLM_DEFAULT_MAX_SUMMARY_TOKENS) or LLM_DEFAULT_MAX_SUMMARY_TOKENS),
),
cfg,
),
extra_body=_extra_body,
debug=settings.debug,
debug=bool(datastore.data['settings']['application'].get('llm_debug', False)),
)
raw, tokens = _resp[0], _resp[1]
input_tokens = _resp[2] if len(_resp) > 2 else 0
@@ -646,7 +597,7 @@ def preview_extract(watch, datastore, content: str) -> dict | None:
Returns {'found': bool, 'answer': str} or None if LLM not configured / no intent.
"""
cfg = _runtime_llm_config(datastore)
cfg = get_llm_config(datastore)
if not cfg:
return None
@@ -660,7 +611,6 @@ def preview_extract(watch, datastore, content: str) -> dict | None:
system_prompt = build_preview_system_prompt()
user_prompt = build_preview_prompt(intent, content, url=url, title=title)
settings = get_llm_settings(datastore)
try:
raw, tokens, *_ = llm_client.completion(
@@ -672,8 +622,8 @@ def preview_extract(watch, datastore, content: str) -> dict | None:
api_key=cfg.get('api_key'),
api_base=cfg.get('api_base'),
max_tokens=apply_local_token_multiplier(JSON_RESPONSE_MAX_TOKENS, cfg),
extra_body=_thinking_extra_body(cfg['model'], settings.thinking_budget),
debug=settings.debug,
extra_body=_thinking_extra_body(cfg['model'], int(datastore.data['settings']['application'].get('llm_thinking_budget', LLM_DEFAULT_THINKING_BUDGET) or 0)),
debug=bool(datastore.data['settings']['application'].get('llm_debug', False)),
)
accumulate_global_tokens(datastore, tokens, model=cfg['model'])
result = parse_preview_response(raw)
@@ -698,7 +648,7 @@ def evaluate_change(watch, datastore, diff: str, current_snapshot: str = '') ->
Results are cached by (intent, diff) hash each unique diff is evaluated exactly once.
"""
cfg = _runtime_llm_config(datastore)
cfg = get_llm_config(datastore)
if not cfg:
return None
@@ -747,7 +697,6 @@ def evaluate_change(watch, datastore, diff: str, current_snapshot: str = '') ->
title=title,
)
settings = get_llm_settings(datastore)
try:
_resp = llm_client.completion(
model=cfg['model'],
@@ -758,8 +707,8 @@ def evaluate_change(watch, datastore, diff: str, current_snapshot: str = '') ->
api_key=cfg.get('api_key'),
api_base=cfg.get('api_base'),
max_tokens=apply_local_token_multiplier(JSON_RESPONSE_MAX_TOKENS, cfg),
extra_body=_thinking_extra_body(cfg['model'], settings.thinking_budget),
debug=settings.debug,
extra_body=_thinking_extra_body(cfg['model'], int(datastore.data['settings']['application'].get('llm_thinking_budget', LLM_DEFAULT_THINKING_BUDGET) or 0)),
debug=bool(datastore.data['settings']['application'].get('llm_debug', False)),
)
raw, tokens = _resp[0], _resp[1]
input_tokens = _resp[2] if len(_resp) > 2 else 0
+3 -3
View File
@@ -2,6 +2,7 @@ from os import getenv
from copy import deepcopy
from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES, RSS_CONTENT_FORMAT_DEFAULT
from changedetectionio.llm.evaluator import LLM_DEFAULT_MAX_SUMMARY_TOKENS, LLM_DEFAULT_THINKING_BUDGET
from changedetectionio.model.Tags import TagsDict
from changedetectionio.notification import (
@@ -70,9 +71,8 @@ class model(dict):
'shared_diff_access': False,
'strip_ignored_lines': False,
'tags': None, # Initialized in __init__ with real datastore_path
# All LLM settings now live nested under application.llm.* (post-migration update_31).
# Defaults come from LLMSettings.model_validate({}) at read time —
# no need to pre-seed an empty {} here.
'llm_thinking_budget': LLM_DEFAULT_THINKING_BUDGET,
'llm_max_summary_tokens': LLM_DEFAULT_MAX_SUMMARY_TOKENS,
'webdriver_delay': None , # Extra delay in seconds before extracting text
'ui': {
'use_page_title_in_list': True,
-65
View File
@@ -1,65 +0,0 @@
"""
Validation/typing layer for the LLM config dict stored at
datastore.data['settings']['application']['llm']
Storage stays a plain dict (orjson-serialized). This model is hydrated on read
(model_validate) and dumped on write (model_dump). WTForms field names match
the storage field names exactly no aliases needed.
"""
from typing import ClassVar, Tuple
from pydantic import BaseModel, ConfigDict
LLM_DEFAULT_THINKING_BUDGET = 0
LLM_DEFAULT_MAX_SUMMARY_TOKENS = 3000
LLM_DEFAULT_LOCAL_TOKEN_MULTIPLIER = 5
LLM_DEFAULT_MAX_INPUT_CHARS = 100_000
LLM_DEFAULT_BUDGET_ACTION = 'skip_llm'
class LLMSettings(BaseModel):
# extra='forbid' rejects any key that isn't a declared field with a
# ValidationError. Loud failure forces new form fields to be declared here
# before they can land in storage — closes the CWE-915 mass-assignment class
# of bugs (see GHSA-h3x5-5j56-hm2j for the canonical example).
model_config = ConfigDict(extra='forbid')
enabled: bool = True
debug: bool = False
override_diff_with_summary: bool = True
restock_use_fallback_extract: bool = True
thinking_budget: int = LLM_DEFAULT_THINKING_BUDGET
max_summary_tokens: int = LLM_DEFAULT_MAX_SUMMARY_TOKENS
budget_action: str = LLM_DEFAULT_BUDGET_ACTION
change_summary_default: str = ''
token_budget_month: int = 0
max_input_chars: int = LLM_DEFAULT_MAX_INPUT_CHARS
# Per-watch per-period token cap; read by _check_token_budget() in evaluator.py.
# 0 means unlimited. Once a watch's usage within the current period hits this cap,
# AI evaluation is skipped for it until the period rolls over. Period is currently
# hard-coded to month (matches the global counter rollover); name is period-agnostic
# to leave room for a configurable period (day/week/month) later.
max_tokens_per_count_period: int = 0
model: str = ''
api_key: str = ''
api_base: str = ''
provider_kind: str = ''
local_token_multiplier: int = LLM_DEFAULT_LOCAL_TOKEN_MULTIPLIER
tokens_total_cumulative: int = 0
tokens_this_month: int = 0
tokens_month_key: str = ''
cost_usd_total_cumulative: float = 0.0
cost_usd_this_month: float = 0.0
# Provider-connection fields wiped on /llm/clear and when the model is emptied.
CONNECTION_FIELDS: ClassVar[Tuple[str, ...]] = (
'model', 'api_key', 'api_base', 'provider_kind', 'local_token_multiplier',
)
# Runtime-managed counters — form submissions must never overwrite these.
PROTECTED_FIELDS: ClassVar[Tuple[str, ...]] = (
'tokens_total_cumulative', 'tokens_this_month', 'tokens_month_key',
'cost_usd_total_cumulative', 'cost_usd_this_month',
)
@@ -1,239 +0,0 @@
# Pydantic Migration
Plan for incrementally moving the app's storage dicts behind Pydantic models. Driven by
security (CWE-915 mass-assignment, see [GHSA-h3x5-5j56-hm2j][advisory]) and schema
enforcement, not just type tidying.
[advisory]: https://github.com/dgtlmoon/changedetection.io/security/advisories/GHSA-h3x5-5j56-hm2j
## The goal
Every form/API endpoint that mutates a stored dict should validate input against a
declared schema before writing. `extra='forbid'` rejects unknown keys — so an attacker
POSTing extra fields like `uuid=…`, `last_checked=…`, `history=[…]` can't smuggle them
into storage. Per-route allowlists work but rot; one declared schema per stored shape
doesn't.
## Prefer a migration over permanent complexity
If you're about to add a compatibility shim, an alias, a backward-compat fallback, or a
"handle both old and new shape" branch — stop and ask whether a one-time `update_N`
migration solves the same problem by *renaming the stored data*. A migration runs once
per install; the shim lives in the code forever and every future contributor has to
understand it.
Concrete example from this PR: the original design used `Field(alias='llm_X')` so
Pydantic could accept both the legacy form-field name (`llm_model`) and the new
storage name (`model`). That alias survived every read/write for the life of the app
and introduced a subtle `model_dump(by_alias=True)` merge bug. The simpler answer was
to rename the form fields to match the storage names (an in-PR rename, no migration
needed since storage was new), drop the aliases entirely, and delete ~25 lines of
plumbing. **Pay once with a migration; don't pay forever with complexity.**
Same principle applies the moment you find yourself writing `dict.get(new_key) or
dict.get(old_key)`. That's a migration in disguise — write the migration instead.
## Architecture choice: validator at the boundary, not domain model
There are two ways to use Pydantic. Pick one per slice — they are not interchangeable.
**Pydantic-as-validator (what we do).** Storage stays a plain dict. A `BaseModel`
validates input at the boundary, dumps back to a dict. No call-site changes; the
existing `watch['x']` dict access keeps working everywhere.
**Pydantic-as-domain-model.** Replace `dict` inheritance with `BaseModel`. ~190 call
sites switch from `watch['x']` to `watch.x`. Much bigger blast radius, defers the
security win. Not what we're doing right now.
The CWE-915 fix only needs the validator pattern. Domain-model replacement is a
separate, later project.
## The template (LLMSettings)
The first migrated slice. Use as the reference for the next one.
**Match the WTForms field names to the storage / Pydantic field names** so the
form-input dict and the storage dict have the same key shape. No aliases, no
`populate_by_name=True`, no `by_alias=True` merge gymnastics. Only reach for
`Field(alias=…)` if you genuinely cannot rename the form field (rare).
`model/LLMSettings.py`:
```python
class LLMSettings(BaseModel):
model_config = ConfigDict(extra='forbid')
enabled: bool = True
model: str = ''
...
# System-managed counters
tokens_total_cumulative: int = 0
...
# Field groups
CONNECTION_FIELDS: ClassVar[Tuple[str, ...]] = ('model', 'api_key', ...)
PROTECTED_FIELDS: ClassVar[Tuple[str, ...]] = ('tokens_total_cumulative', ...)
```
Boundary pattern at the route handler:
```python
# Read
settings = LLMSettings.model_validate(
datastore.data['settings']['application'].get('llm') or {}
)
# Merge form input
form_input = dict(form.data.get('llm') or {})
for protected in LLMSettings.PROTECTED_FIELDS:
form_input.pop(protected, None) # counters never come from form
merged = LLMSettings.model_validate({**settings.model_dump(), **form_input})
# Write — re-validates the schema on every write
datastore.data['settings']['application']['llm'] = merged.model_dump()
```
## Unresolved architectural decisions
Two decisions need answers before the `WatchInput` slice. They're not blockers for `App.py`.
### OpenAPI spec vs Pydantic model — who's source of truth?
Today: `docs/api-spec.yaml` declares the Watch/Tag shape; `model/schema_utils.py` reads
it to compute readonly fields; the API layer validates against it; the model layer is a
plain dict that doesn't know about either. When `WatchInput` lands, that's a third
shape declaration.
Two ways to live:
- **Pydantic is source.** Generate / sync `api-spec.yaml` from the model
(e.g. via `model_json_schema()`). One declaration, multiple consumers. Long-term
right answer; needs tooling.
- **Parallel sources with discipline.** Hand-keep them aligned. Faster to ship but
drift is inevitable — that's the bug class we're already trying to close.
Recommendation: start parallel (keep `api-spec.yaml` for now), but write Watch's
Pydantic model so it could be the eventual single source. Don't *invent* a new
field shape — match the spec.
### Plugin / processor_config_* extensibility
`processor_config_restock_diff` (and future processor configs) are written by
plugins, not the core. `extra='forbid'` on a Watch input model would reject them.
Options:
- **Per-processor sub-models.** Each plugin owns its `<Processor>Settings` Pydantic
model; Watch input validates only core fields, processor configs validate
separately at their own boundary (the per-watch `restock_diff.json`, etc.).
- **Opaque pass-through.** Watch input model treats `processor_config_*` as a
declared dict-typed field. Loses per-key validation but preserves the
plugin-extensibility contract.
Recommendation: per-processor sub-models. Matches the file split already done in
`update_30` (separate `restock_diff.json` per watch).
## Migration order
| Target | Difficulty | Value | Status |
|---|---|---|---|
| `LLMSettings` | low | medium | done (this PR) |
| `App.py``AppSettings` (nested) | low | medium | next |
| `WatchInput` (form/API validator) | medium | **HIGH — closes [GHSA-h3x5-5j56-hm2j][advisory]** | next-next |
| `TagInput` (form/API validator) | medium | medium | after Watch |
| `watch_base(dict)``BaseModel` | very high | high | separate multi-PR project, much later |
`Tags.py` (TagsDict), `persistence.py`, `schema_utils.py` are not data models — leave alone.
### Concrete next steps
1. **`App.py`.** Pure dict tree under `settings.{application,requests,headers}`. Define
nested `BaseModel`s; `LLMSettings` slots in as the existing sub-tree. No call-site
churn — just the global settings POST handler. Sets the pattern for nested models.
2. **`WatchInput` BaseModel** for `blueprint/ui/edit.py:225` and `api/Watch.py`. Replace:
```python
datastore.data['watching'][uuid].update(form.data) # CWE-915
```
with:
```python
validated = WatchInput.model_validate(form.data)
datastore.data['watching'][uuid].update(validated.model_dump())
```
Closes the unpatched advisory. Should be a security-tagged commit referencing the GHSA.
3. **`TagInput` BaseModel** — same pattern, smaller.
## Gotchas discovered
These cost real debugging time in the LLMSettings PR. Worth knowing before the next slice.
### `extra='forbid'` is the right default
`extra='ignore'` silently drops unknowns and hides developer mistakes (add a form field,
forget to declare it on the model, your feature appears to work until you reload). `forbid`
fails loudly. `allow` defeats the purpose entirely — it's how injection succeeds.
### Don't use Field aliases unless you actually need them
The LLMSettings PR originally used `alias='llm_X'` to bridge llm_-prefixed WTForms
names to stripped storage names. That created a documented gotcha: with
`extra='forbid'`, having both `model` and `llm_model` in the same input dict is a
`ValidationError`, and merging existing-storage-dump with form input required
`by_alias=True` to keep both sides on the alias shape. We fixed it by renaming the
form fields to match the storage field names. **Match the form to the model
upfront and you avoid the whole class of merge bugs.**
### Round-trip counters through the model, don't mutate the dict
If runtime code (e.g. a token accumulator) writes to the storage dict directly, the
schema is bypassed. Load → mutate instance attributes → `model_dump()` → write back.
This re-validates on every write and prevents drift.
### Per-call validation needs strict + tolerant modes? Don't.
You might be tempted to validate form input strictly but allow extras in storage
hydration. Don't — `extra='forbid'` everywhere means storage drift is impossible. If
something put unknown keys in storage, you want loud failure, not silent acceptance.
### Migrations are convention-based by accident if you let them be
`for k in list(d) if k.startswith('llm_')` is shorter than an explicit list but
silently catches any future flat `llm_*` key. Migrations are forever — prefer an
explicit allowlist of keys to move, even if it's verbose.
## What NOT to do
- Don't add custom helper methods (`dump_without_connection()`, `clear_X()`) when stock
`model_dump(exclude=set(FIELDS))` works. The standard idiom is more readable and
zero-line.
- Don't push security/business logic into the model (e.g. SSRF guards, credential-exfil
checks). The model owns field shape and validation. Route handlers own
policy. Mixing them dilutes both.
- Don't make `get_X_config()` return a Pydantic instance if callers do dict-style access.
Either migrate all call sites (high-touch) or keep returning a dict and let the model
be the validation/dump layer only.
- Don't `model_copy(update=...)` without re-validating. It doesn't coerce types or
enforce `extra='forbid'`. Use `model_validate({**old.model_dump(), **updates})` for
strict merges.
## Required for each new slice
Each migration PR should ship:
- `model/<Thing>Settings.py` (or input model) — declared schema, `extra='forbid'`,
field aliases if there's a name mismatch between form and storage.
- `store/updates.py:update_N` if the storage shape changes. Pure dict-shuffling, no
Pydantic import (migrations should not depend on the model — model evolves
independently).
- `tests/unit/test_<thing>.py` — unit coverage of the model itself: defaults,
alias merge, type coercion, `extra='forbid'` rejection, dump shapes.
- All runtime callers updated to go through `get_<thing>_settings(datastore)` or
equivalent, not raw dict reads.
## Reference
- `model/LLMSettings.py` — the template
- `tests/unit/test_llm_settings.py` — model unit-test template
- `store/updates.py:update_31` — schema migration template
- `blueprint/settings/__init__.py` (POST handler) — boundary-validation template
- `llm/evaluator.py:accumulate_global_tokens` — instance-mutation-then-dump-back template
+1 -2
View File
@@ -376,8 +376,7 @@ def process_notification(n_object: NotificationContextData, datastore):
# AI Change Summary: optionally replace {{ diff }} with the AI summary
_llm_change_summary = (n_object.get('_llm_change_summary') or '').strip()
from changedetectionio.llm.evaluator import get_llm_settings
_override_diff = get_llm_settings(datastore).override_diff_with_summary
_override_diff = datastore.data['settings']['application'].get('llm_override_diff_with_summary', True)
if _llm_change_summary and _override_diff:
n_object['diff'] = _llm_change_summary
@@ -204,8 +204,6 @@ class NotificationContextData(dict):
'diff_changed_from': FormattableExtract('', '', extract_fn=lambda x: x),
'diff_changed_to': FormattableExtract('', '', extract_fn=lambda x: x),
'diff_url': None,
# Always the raw +/- diff regardless of LLM summary override (populated in handler.py from {{diff}})
'raw_diff': FormattableDiff('', ''),
'markup_text_links_to_html_links': False, # If automatic conversion of plaintext to HTML should happen
'notification_timestamp': time.time(),
'prev_snapshot': None,
@@ -196,23 +196,22 @@ def get_itemprop_availability_override(content, fetcher_name, fetcher_instance,
logger.debug("LLM restock fallback: no datastore injected yet, skipping")
return None
# Gate on the user setting (default True — enabled out of the box)
app_settings = datastore.data.get('settings', {}).get('application', {})
if not app_settings.get('llm_restock_use_fallback_extract', True):
logger.debug("LLM restock fallback: disabled in settings")
return None
try:
from changedetectionio.llm.evaluator import _runtime_llm_config, accumulate_global_tokens, get_llm_settings
from changedetectionio.llm.evaluator import get_llm_config, accumulate_global_tokens
from changedetectionio.llm import client as llm_client
except ImportError as e:
logger.debug(f"LLM restock fallback: LLM libraries not available ({e})")
return None
# Gate on the user setting (default True — enabled out of the box)
if not get_llm_settings(datastore).restock_use_fallback_extract:
logger.debug("LLM restock fallback: disabled in settings")
return None
# _runtime_llm_config returns None (with a debug log) when the master 'llm_enabled'
# toggle is off, so this path is gated for free.
llm_cfg = _runtime_llm_config(datastore)
llm_cfg = get_llm_config(datastore)
if not llm_cfg or not llm_cfg.get('model'):
logger.debug("LLM restock fallback: no LLM model configured or LLM disabled, skipping")
logger.debug("LLM restock fallback: no LLM model configured, skipping")
return None
text_content = _strip_html(content) if content else ''
@@ -35,50 +35,6 @@ def _task(watch, update_handler):
return text_after_filter
def _compute_ignore_line_numbers_for_preview(text_pre_extract, ignore_patterns, extract_patterns):
"""1-indexed output line numbers in the post-extract display that correspond
to input lines matching ignore_text patterns.
Needed because extract_text (#4138) transforms line content — e.g. "0.54.10"
becomes ".54.10" so a substring match for "0.54.10" against the post-extract
text fails and the preview UI can no longer mark the line as ignored. We find
the ignored line numbers in the pre-extract text and replay extract_by_regex
line-by-line to map them forward.
"""
from changedetectionio import html_tools
from changedetectionio.processors.text_json_diff.processor import ContentTransformer
if not text_pre_extract or not ignore_patterns:
return []
ignored_input_lines = set(
html_tools.strip_ignore_text(
content=text_pre_extract,
wordlist=ignore_patterns,
mode='line numbers'
)
)
if not ignored_input_lines:
return []
if not extract_patterns:
return sorted(ignored_input_lines)
# Replay extract_by_regex per-line. Each emitted match ends with exactly one
# '\n', so counting newlines tells us how many output lines this input produced.
output_line_counter = 0
result = []
for input_idx, line in enumerate(text_pre_extract.splitlines()):
is_ignored = (input_idx + 1) in ignored_input_lines
matches_in_line = ContentTransformer.extract_by_regex(line, extract_patterns).count('\n')
for _ in range(matches_in_line):
output_line_counter += 1
if is_ignored:
result.append(output_line_counter)
return result
def prepare_filter_prevew(datastore, watch_uuid, form_data):
'''Used by @app.route("/edit/<uuid_str:uuid>/preview-rendered", methods=['POST'])'''
from changedetectionio import forms, html_tools
@@ -94,7 +50,6 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
text_after_filter = ''
text_before_filter = ''
text_pre_extract = ''
trigger_line_numbers = []
ignore_line_numbers = []
blocked_line_numbers = []
@@ -134,22 +89,15 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
update_handler.fetcher.content = str(decompressed_data) # str() because playwright/puppeteer/requests return string
update_handler.fetcher.headers['content-type'] = tmp_watch.get('content-type')
# Process our watch with filters and the HTML from disk, and also a blank watch with no filters but also with the same HTML from disk.
# The third task runs with extract_text cleared so we can compute ignore_line_numbers
# against the pre-extract text (extract_text transforms lines so post-extract substring
# matching for ignore patterns would otherwise fail — see #4138 follow-up).
# Process our watch with filters and the HTML from disk, and also a blank watch with no filters but also with the same HTML from disk
# Do this as parallel threads (not processes) to avoid pickle issues with Lock objects
tmp_watch_no_extract = deepcopy(tmp_watch)
tmp_watch_no_extract['extract_text'] = []
try:
with ThreadPoolExecutor(max_workers=3) as executor:
with ThreadPoolExecutor(max_workers=2) as executor:
future1 = executor.submit(_task, tmp_watch, update_handler)
future2 = executor.submit(_task, blank_watch_no_filters, update_handler)
future3 = executor.submit(_task, tmp_watch_no_extract, update_handler)
text_after_filter = future1.result()
text_before_filter = future2.result()
text_pre_extract = future3.result()
except Exception as e:
x=1
@@ -163,11 +111,10 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
try:
text_to_ignore = tmp_watch.get('ignore_text', []) + datastore.data['settings']['application'].get('global_ignore_text', [])
ignore_line_numbers = _compute_ignore_line_numbers_for_preview(
text_pre_extract=text_pre_extract,
ignore_patterns=text_to_ignore,
extract_patterns=tmp_watch.get('extract_text', [])
)
ignore_line_numbers = html_tools.strip_ignore_text(content=text_after_filter,
wordlist=text_to_ignore,
mode='line numbers'
)
except Exception as e:
text_before_filter = f"Error: {str(e)}"
@@ -217,10 +217,8 @@ def render(watch, datastore, request, url_for, render_template, flash, redirect,
llm_summary_prompt = _prompt
# Must match the cache_prompt the worker writes and the UI ajax route reads —
# using UI default diff prefs so the initial render finds the worker's pre-cache.
from changedetectionio.llm.evaluator import get_llm_settings
_ls = get_llm_settings(datastore)
_max_summary_tokens = _ls.max_summary_tokens
_llm_model = _ls.model
_max_summary_tokens = datastore.data['settings']['application'].get('llm_max_summary_tokens', 3000)
_llm_model = (datastore.data['settings']['application'].get('llm') or {}).get('model', '')
_cache_prompt = build_summary_cache_prompt(
effective_prompt=_prompt,
max_summary_tokens=_max_summary_tokens,
@@ -9,10 +9,6 @@ function request_textpreview_update() {
$('textarea:visible, input:visible').each(function () {
const $element = $(this); // Cache the jQuery object for the current element
const name = $element.attr('name'); // Get the name attribute of the element
// Radios share a name across multiple inputs; .val() returns the value
// attribute regardless of checked state, so iterating would let the last
// unchecked radio overwrite the user's actual selection. Skip unchecked.
if ($element.is(':radio') && !$element.is(':checked')) return;
data[name] = $element.is(':checkbox') ? ($element.is(':checked') ? $element.val() : false) : $element.val();
});
-50
View File
@@ -775,53 +775,3 @@ class DatastoreUpdatesMixin:
tag.commit()
logger.info(f"update_30: migrated tag {tag_uuid} restock_settings → processor_config_restock_diff")
def update_31(self):
"""Fold any flat application.llm_* key into nested application.llm.<stripped>.
Before: a handful of LLM settings (llm_enabled, llm_thinking_budget, ) lived
directly on settings.application alongside everything else, while the provider
config (model, api_key, ) was already nested under settings.application.llm.
Unifies them under one parent so the LLMSettings pydantic model has a single
home to read/write.
Flat key wins on conflict (most-recent form-saved value). Idempotent.
"""
application = self.data['settings']['application']
present = [k for k in list(application) if k.startswith('llm_')]
if not present:
return
nested = application.get('llm') or {}
for flat in present:
nested[flat.removeprefix('llm_')] = application.pop(flat)
application['llm'] = nested
logger.info(f"update_31: folded {len(present)} flat llm_* keys into application.llm.* "
f"({', '.join(present)})")
def update_32(self):
"""Drop max_tokens_per_check and rename max_tokens_cumulative → max_tokens_per_count_period.
max_tokens_per_check was never reachable from the UI (form field declared but
never rendered or saved) and overlapped with the cumulative cap. Removing it.
max_tokens_cumulative was misleading the field was used as a per-watch
per-period cap, not lifetime. Renamed so the semantic is clear and so a
future configurable period (day/week/month) doesn't force another rename.
Both keys are unreached from real installs (no UI path on prior releases);
this migration is mostly for branches and devs running pre-release commits.
"""
llm = self.data['settings']['application'].get('llm') or {}
if not llm:
return
changed = False
if 'max_tokens_per_check' in llm:
del llm['max_tokens_per_check']
changed = True
if 'max_tokens_cumulative' in llm:
llm.setdefault('max_tokens_per_count_period', llm.pop('max_tokens_cumulative'))
changed = True
if changed:
self.data['settings']['application']['llm'] = llm
logger.info("update_32: cleaned up obsolete max_tokens_per_check / renamed max_tokens_cumulative")
@@ -112,7 +112,7 @@
<td><code>{{ '{{triggered_text}}' }}</code></td>
<td>{{ _('Text that tripped the trigger from filters') }}</td>
</tr>
{% if not llm_features_disabled and settings_application and settings_application.get('llm', {}).get('model') %}
{% if settings_application and settings_application.get('llm', {}).get('model') %}
<tr>
<td><code>{{ '{{diff}}' }}</code> <small style="opacity:0.6">{{ _('(upgraded)') }}</small></td>
<td>{{ _('When AI Change Summary is configured, contains the AI-generated description instead of the raw diff. Falls back to raw diff when not configured.') }}</td>
-2
View File
@@ -281,7 +281,6 @@
</div>
</dialog>
{% if not llm_features_disabled %}
<!-- LLM Not Configured Modal -->
<dialog id="llm-not-configured-modal" class="modal-dialog" aria-labelledby="llm-not-configured-modal-title">
<div class="modal-header">
@@ -295,7 +294,6 @@
<button type="button" class="pure-button" id="close-llm-not-configured-modal">{{ _('Close') }}</button>
</div>
</dialog>
{% endif %}
<!-- Search Modal -->
{% if current_user.is_authenticated or not has_password %}
-2
View File
@@ -37,12 +37,10 @@
</li>
{% endif %}
<li class="pure-menu-item menu-collapsible" id="inline-menu-extras-group">
{% if not llm_features_disabled %}
<button class="toggle-button toggle-ai-mode" type="button" title="{{ _('Toggle AI Mode') }}" data-llm-configured="{{ 'true' if llm_configured else 'false' }}" data-llm-settings-url="{{ url_for('settings.settings_page') }}#ai">
<span class="visually-hidden">{{ _('Toggle AI mode') }}</span>
{% include "svgs/ai-mode-icon.svg" %}<span class="ai-mode-label">LLM</span>
</button>
{% endif %}
<button class="toggle-button toggle-light-mode " type="button" title="{{ _('Toggle Light/Dark Mode') }}">
<span class="visually-hidden">{{ _('Toggle light/dark mode') }}</span>
<span class="icon-light">
+58 -45
View File
@@ -294,82 +294,78 @@ class TestTokenBudget:
assert _check_token_budget(watch, cfg, tokens_this_call=10_000) is True
def test_per_period_limit_exceeded_returns_false(self):
"""Per-period tokens exceeding the cap → False."""
from changedetectionio.llm.evaluator import _check_token_budget, _get_month_key
def test_per_check_limit_exceeded_returns_false(self):
"""Tokens on this call exceeding per-check limit → False."""
from changedetectionio.llm.evaluator import _check_token_budget
watch = _make_watch()
watch['llm_tokens_this_period'] = 900
watch['llm_tokens_period_key'] = _get_month_key()
cfg = {'max_tokens_per_count_period': 1000}
cfg = {'max_tokens_per_check': 100}
result = _check_token_budget(watch, cfg, tokens_this_call=150)
assert result is False
def test_per_check_limit_not_exceeded_returns_true(self):
"""Tokens on this call within per-check limit → True."""
from changedetectionio.llm.evaluator import _check_token_budget
watch = _make_watch()
cfg = {'max_tokens_per_check': 200}
result = _check_token_budget(watch, cfg, tokens_this_call=150)
assert result is True
def test_cumulative_limit_exceeded_returns_false(self):
"""Total accumulated tokens exceeding cumulative limit → False."""
from changedetectionio.llm.evaluator import _check_token_budget
watch = _make_watch()
watch['llm_tokens_used_cumulative'] = 900
cfg = {'max_tokens_cumulative': 1000}
# This call adds 200 → total 1100 > 1000
result = _check_token_budget(watch, cfg, tokens_this_call=200)
assert result is False
def test_per_period_limit_not_yet_exceeded_returns_true(self):
"""Per-period tokens within the cap → True."""
from changedetectionio.llm.evaluator import _check_token_budget, _get_month_key
def test_cumulative_limit_not_yet_exceeded_returns_true(self):
"""Total accumulated tokens within cumulative limit → True."""
from changedetectionio.llm.evaluator import _check_token_budget
watch = _make_watch()
watch['llm_tokens_this_period'] = 500
watch['llm_tokens_period_key'] = _get_month_key()
cfg = {'max_tokens_per_count_period': 1000}
watch['llm_tokens_used_cumulative'] = 500
cfg = {'max_tokens_cumulative': 1000}
result = _check_token_budget(watch, cfg, tokens_this_call=100)
assert result is True
def test_period_rollover_zeroes_counter(self):
"""Stale period_key triggers rollover: counter resets before this call's tokens are added."""
from changedetectionio.llm.evaluator import _check_token_budget, _get_month_key
watch = _make_watch()
watch['llm_tokens_this_period'] = 999_999 # last period's giant total
watch['llm_tokens_period_key'] = '1970-01' # ancient — guaranteed stale
cfg = {'max_tokens_per_count_period': 1000}
# This call adds 100 → after rollover should be 100, under the 1000 cap
result = _check_token_budget(watch, cfg, tokens_this_call=100)
assert result is True
assert watch['llm_tokens_this_period'] == 100
assert watch['llm_tokens_period_key'] == _get_month_key()
def test_tokens_accumulated_into_both_counters(self):
"""tokens_this_call increments both the lifetime stat and the per-period counter."""
from changedetectionio.llm.evaluator import _check_token_budget, _get_month_key
def test_tokens_accumulated_into_watch(self):
"""tokens_this_call is added to watch['llm_tokens_used_cumulative']."""
from changedetectionio.llm.evaluator import _check_token_budget
watch = _make_watch()
watch['llm_tokens_used_cumulative'] = 300
watch['llm_tokens_this_period'] = 50
watch['llm_tokens_period_key'] = _get_month_key()
cfg = {}
_check_token_budget(watch, cfg, tokens_this_call=75)
assert watch['llm_tokens_used_cumulative'] == 375
assert watch['llm_tokens_this_period'] == 125
def test_zero_tokens_call_does_not_change_counters(self):
"""Calling with tokens_this_call=0 (pre-flight check) doesn't modify counters."""
from changedetectionio.llm.evaluator import _check_token_budget, _get_month_key
def test_zero_tokens_call_does_not_change_cumulative(self):
"""Calling with tokens_this_call=0 (pre-flight check) doesn't modify cumulative."""
from changedetectionio.llm.evaluator import _check_token_budget
watch = _make_watch()
watch['llm_tokens_used_cumulative'] = 200
watch['llm_tokens_this_period'] = 80
watch['llm_tokens_period_key'] = _get_month_key()
cfg = {}
_check_token_budget(watch, cfg, tokens_this_call=0)
assert watch['llm_tokens_used_cumulative'] == 200
assert watch['llm_tokens_this_period'] == 80
def test_evaluate_change_skips_call_when_per_period_over_budget(self):
"""Pre-flight check: if already over the period cap, skip the LLM call and fail open."""
from changedetectionio.llm.evaluator import evaluate_change, _get_month_key
def test_evaluate_change_skips_call_when_cumulative_over_budget(self):
"""Pre-flight cumulative check: if already over budget, skip LLM call and fail open."""
from changedetectionio.llm.evaluator import evaluate_change
ds = _make_datastore(llm_cfg={'model': 'gpt-4o-mini', 'max_tokens_per_count_period': 100})
ds = _make_datastore(llm_cfg={'model': 'gpt-4o-mini', 'max_tokens_cumulative': 100})
watch = _make_watch(llm_intent='flag price drops')
watch['llm_tokens_this_period'] = 500 # already far over
watch['llm_tokens_period_key'] = _get_month_key()
watch['llm_tokens_used_cumulative'] = 500 # already far over
with patch('changedetectionio.llm.client.completion') as mock_llm:
result = evaluate_change(watch, ds, diff='- $500\n+ $400')
@@ -378,6 +374,23 @@ class TestTokenBudget:
# Fail open: important=True so the notification is NOT suppressed
assert result == {'important': True, 'summary': ''}
def test_evaluate_change_per_check_limit_fails_open(self):
"""Per-check token exceeded after call → result still returned (fail open)."""
from changedetectionio.llm.evaluator import evaluate_change
# max_tokens_per_check is 50, but the call returns 150 tokens
ds = _make_datastore(llm_cfg={'model': 'gpt-4o-mini', 'max_tokens_per_check': 50})
watch = _make_watch(llm_intent='flag price drops')
llm_response = '{"important": false, "summary": "Only minor change"}'
with patch('changedetectionio.llm.client.completion', return_value=(llm_response, 150)):
result = evaluate_change(watch, ds, diff='- $500\n+ $499')
# LLM said not important, but even with per-check warning the result is returned
# (budget warning is logged but evaluation result is still used)
assert result is not None
assert 'important' in result
# ---------------------------------------------------------------------------
# resolve_llm_field (generic cascade)
@@ -1,62 +0,0 @@
"""
Smoke test for the LLM_FEATURES_DISABLED env var.
The env var is intended to hide every LLM/AI surface (settings tab, edit tab,
base-template AI toggle/modal) for hosted deployments. This test renders the
three primary pages with the env var set and verifies that none of the
LLM-related markers leak through.
"""
from flask import url_for
def _llm_markers_absent(body: bytes, where: str = ''):
"""All of these strings appear in LLM UI surfaces — none should render."""
for marker in (b'AI / LLM', b'toggle-ai-mode', b'llm-not-configured-modal',
b'id="ai-llm"', b'#ai-llm', b'href="#ai"'):
if marker in body:
idx = body.find(marker)
context = body[max(0, idx - 80):idx + len(marker) + 80].decode('utf-8', 'replace')
raise AssertionError(f"[{where}] {marker!r} found in body, context: ...{context}...")
def test_llm_features_disabled_hides_ui(client, live_server, monkeypatch):
monkeypatch.setenv('LLM_FEATURES_DISABLED', 'true')
# Sanity: helper reports the env var is in effect
from changedetectionio.llm.evaluator import is_llm_features_disabled, get_llm_config
assert is_llm_features_disabled() is True
# get_llm_config() must return None so every `if llm_configured` template hides
datastore = client.application.config.get('DATASTORE')
assert get_llm_config(datastore) is None
# 1. Watch list (base.html + menu.html surface)
res = client.get(url_for('watchlist.index'))
assert res.status_code == 200
_llm_markers_absent(res.data, where='watchlist')
# 2. Settings page (should not have an AI / LLM tab or the LLM tab body)
res = client.get(url_for('settings.settings_page'))
assert res.status_code == 200
_llm_markers_absent(res.data, where='settings')
# 3. Edit page for a watch (should not have an AI / LLM tab or include_llm_intent body)
uuid = datastore.add_watch(url='http://example.com', extras={'title': 'Disabled LLM watch'})
res = client.get(url_for('ui.ui_edit.edit_page', uuid=uuid))
assert res.status_code == 200
_llm_markers_absent(res.data, where='edit')
# The watch-edit-only intent textarea should also be absent
assert b'name="llm_intent"' not in res.data
assert b'name="llm_change_summary"' not in res.data
def test_llm_features_enabled_by_default(client, live_server, monkeypatch):
"""When LLM_FEATURES_DISABLED is unset, the AI / LLM surfaces are still rendered."""
monkeypatch.delenv('LLM_FEATURES_DISABLED', raising=False)
from changedetectionio.llm.evaluator import is_llm_features_disabled
assert is_llm_features_disabled() is False
res = client.get(url_for('settings.settings_page'))
assert res.status_code == 200
# The AI / LLM settings tab anchor should be present when not disabled
assert b'href="#ai"' in res.data
@@ -14,9 +14,8 @@ def _make_datastore(llm_model='gpt-4o-mini', enabled=True):
ds.data = {
'settings': {
'application': {
'llm_restock_use_fallback_extract': enabled,
'llm': {
'enabled': True,
'restock_use_fallback_extract': enabled,
'model': llm_model,
'api_key': 'test-key',
'api_base': '',
@@ -85,8 +84,8 @@ class TestLLMRestockPluginDisabled:
ds.data = {
'settings': {
'application': {
# No 'llm' key → get_llm_config returns None;
# restock_use_fallback_extract still defaults to True via LLMSettings
'llm_restock_use_fallback_extract': True,
# No 'llm' key → get_llm_config returns None
}
}
}
@@ -77,82 +77,3 @@ def test_content_filter_live_preview(client, live_server, measure_memory_usage,
assert reply.get('trigger_line_numbers') == [1] # Triggers "Awesome" in line 1
delete_all_watches(client)
def _setup_version_list_preview(datastore_path, client):
"""Shared HTML fixture for #4138 preview regressions (version tag list)."""
import time
data = """<html><body>
0.55.5<br>
0.55.4<br>
0.55.3<br>
0.54.10<br>
0.54.9<br>
</body></html>"""
with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
f.write(data)
test_url = url_for('test_endpoint', _external=True)
uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
time.sleep(0.5)
wait_for_all_checks(client)
return test_url, uuid
def test_preview_ignore_highlight_with_extract_text(client, live_server, measure_memory_usage, datastore_path):
"""Regression for #4138 follow-up: when extract_text rewrites a line (e.g. "0.54.10"".54.10"),
the preview must still highlight that row as 'ignored' even though substring matching against the
post-extract text fails."""
import json
test_url, uuid = _setup_version_list_preview(datastore_path, client)
res = client.post(
url_for("ui.ui_edit.watch_get_preview_rendered", uuid=uuid),
data={
"include_filters": "",
"fetch_backend": 'html_requests',
"ignore_text": "0.54.10",
"extract_text": r"/(.\d+\.\d+)/",
"url": test_url,
},
)
reply = json.loads(res.data.decode('utf-8'))
# The regex strips the leading "0", so the post-extract line for the ignored input is ".54.10".
# The preview should still mark its position (line 4) as ignored.
assert reply.get('ignore_line_numbers') == [4], \
f"Expected line 4 to be highlighted as ignored, got {reply.get('ignore_line_numbers')!r}"
delete_all_watches(client)
def test_preview_strip_ignored_lines_with_extract_text(client, live_server, measure_memory_usage, datastore_path):
"""Regression for #4138 follow-up: with strip_ignored_lines enabled, an ignored line must be
removed from the preview output even when extract_text would otherwise rewrite it (0.54.10 .54.10)."""
import json
test_url, uuid = _setup_version_list_preview(datastore_path, client)
res = client.post(
url_for("ui.ui_edit.watch_get_preview_rendered", uuid=uuid),
data={
"include_filters": "",
"fetch_backend": 'html_requests',
"ignore_text": "0.54.10",
"extract_text": r"/(.\d+\.\d+)/",
"strip_ignored_lines": "true",
"url": test_url,
},
)
reply = json.loads(res.data.decode('utf-8'))
after_filter = reply.get('after_filter', '')
assert '.54.10' not in after_filter, \
f"Stripped ignored line should not appear in preview output, got:\n{after_filter!r}"
assert '0.54.10' not in after_filter
assert reply.get('ignore_line_numbers') == [], \
f"Stripped lines need no highlight, got {reply.get('ignore_line_numbers')!r}"
delete_all_watches(client)
@@ -329,9 +329,9 @@ def test_settings_form_preserves_api_key_when_submitted_blank(
res = client.post(
url_for('settings.settings_page'),
data={
'llm-model': 'gpt-4o',
'llm-api_key': '', # blank — PasswordField behaviour
'llm-api_base': '',
'llm-llm_model': 'gpt-4o',
'llm-llm_api_key': '', # blank — PasswordField behaviour
'llm-llm_api_base': '',
'application-pager_size': '50',
'application-notification_format': 'System default',
'requests-time_between_check-days': '0',
@@ -452,9 +452,9 @@ def test_settings_form_rejects_private_api_base(
res = client.post(
url_for('settings.settings_page'),
data={
'llm-model': 'gpt-4o',
'llm-api_key': '',
'llm-api_base': 'http://127.0.0.1:11434',
'llm-llm_model': 'gpt-4o',
'llm-llm_api_key': '',
'llm-llm_api_base': 'http://127.0.0.1:11434',
'application-pager_size': '50',
'application-notification_format': 'System default',
'requests-time_between_check-days': '0',
@@ -653,23 +653,11 @@ def test_llm_clear_summary_cache_get_does_not_wipe_cache(
def test_llm_clear_via_post_still_works(
client, live_server, measure_memory_usage, datastore_path):
"""Confirm the legit confirm-then-POST flow wipes the provider credentials.
Post-LLMSettings: /llm/clear strips only the connection fields (model, api_key,
api_base, provider_kind, local_token_multiplier). User-set toggles, the global
summary prompt, monthly budgets, and system token counters survive. This matches
the settings-page "empty model" save semantic and the LLMSettings.CONNECTION_FIELDS
grouping see PYDANTIC_MIGRATION.md.
"""
"""Confirm the legit confirm-then-POST flow still wipes LLM config."""
ds = client.application.config.get('DATASTORE')
_configure_llm(ds)
assert ds.data['settings']['application'].get('llm', {}).get('api_key') == CANARY_KEY
res = client.post(url_for('settings.llm.llm_clear'), follow_redirects=True)
assert res.status_code == 200
# The api_key must be gone (this is what the test really cares about).
llm = ds.data['settings']['application'].get('llm') or {}
assert 'api_key' not in llm, f"api_key should have been wiped, got: {llm!r}"
assert 'model' not in llm
assert 'api_base' not in llm
assert 'llm' not in ds.data['settings']['application']
@@ -28,11 +28,7 @@ def _set_response(datastore_path, content):
def _configure_llm(client):
ds = client.application.config.get('DATASTORE')
# Merge into the existing llm dict so other test setup (e.g. change_summary_default
# set via _set_global_default) survives.
existing = ds.data['settings']['application'].get('llm') or {}
existing.update({'model': 'gpt-4o-mini', 'api_key': 'sk-test'})
ds.data['settings']['application']['llm'] = existing
ds.data['settings']['application']['llm'] = {'model': 'gpt-4o-mini', 'api_key': 'sk-test'}
# ---------------------------------------------------------------------------
@@ -242,9 +238,7 @@ def test_llm_summary_ajax_error_displayed_not_silenced(
# ---------------------------------------------------------------------------
def _set_global_default(ds, prompt):
llm = ds.data['settings']['application'].get('llm') or {}
llm['change_summary_default'] = prompt
ds.data['settings']['application']['llm'] = llm
ds.data['settings']['application']['llm_change_summary_default'] = prompt
def test_global_default_used_when_watch_and_tag_have_none(
@@ -335,7 +329,7 @@ def test_hardcoded_fallback_when_nothing_set(
watch['llm_change_summary'] = ''
# Ensure global default is also empty
_set_global_default(ds, '')
ds.data['settings']['application']['llm_change_summary_default'] = ''
assert get_effective_summary_prompt(watch, ds) == DEFAULT_CHANGE_SUMMARY_PROMPT
@@ -397,8 +391,8 @@ def test_llm_summary_ajax_sets_last_viewed(
def test_global_default_saved_and_loaded_via_settings_form(
client, live_server, measure_memory_usage, datastore_path):
"""
Submitting the settings form persists the global default prompt into
application.llm.change_summary_default (single nested home for all LLM settings).
Submitting the settings form persists llm_change_summary_default at
settings.application level (not inside the llm credentials dict).
"""
from changedetectionio.tests.util import live_server_setup
live_server_setup(live_server)
@@ -411,20 +405,21 @@ def test_global_default_saved_and_loaded_via_settings_form(
'application-empty_pages_are_a_change': '',
'requests-time_between_check-minutes': 180,
'application-fetch_backend': 'html_requests',
'llm-change_summary_default': 'Saved global prompt.',
'llm-llm_change_summary_default': 'Saved global prompt.',
# Keep existing model so llm block is retained
'llm-model': 'gpt-4o-mini',
'llm-llm_model': 'gpt-4o-mini',
},
follow_redirects=True,
)
assert b'Settings updated.' in res.data
ds = client.application.config.get('DATASTORE')
llm_dict = ds.data['settings']['application'].get('llm', {})
assert llm_dict.get('change_summary_default') == 'Saved global prompt.', f"Got: {llm_dict!r}"
stored = ds.data['settings']['application'].get('llm_change_summary_default', '')
assert stored == 'Saved global prompt.', f"Got: {stored!r}"
# And the old flat key must not be re-introduced
assert 'llm_change_summary_default' not in ds.data['settings']['application']
# Must NOT be buried inside the llm credentials dict
llm_dict = ds.data['settings']['application'].get('llm', {})
assert 'change_summary_default' not in llm_dict
delete_all_watches(client)
@@ -445,11 +440,7 @@ def test_global_default_survives_llm_clear(
res = client.post(url_for('settings.llm.llm_clear'), follow_redirects=True)
assert res.status_code == 200
llm_dict = ds.data['settings']['application'].get('llm') or {}
assert llm_dict.get('change_summary_default') == 'Surviving prompt.'
# The credential fields should be gone
assert 'model' not in llm_dict
assert 'api_key' not in llm_dict
assert ds.data['settings']['application'].get('llm_change_summary_default') == 'Surviving prompt.'
delete_all_watches(client)
@@ -168,9 +168,9 @@ def test_settings_form_preserves_token_counters(
url_for('settings.settings_page'),
data={
# LLM sub-form fields
'llm-model': 'gpt-4o',
'llm-api_key': 'sk-different-key',
'llm-api_base': '',
'llm-llm_model': 'gpt-4o',
'llm-llm_api_key': 'sk-different-key',
'llm-llm_api_base': '',
# Minimal required fields to pass form validation
'application-pager_size': '50',
'application-notification_format': 'System default',
@@ -209,9 +209,9 @@ def test_settings_form_cannot_inject_fake_token_counts(
res = client.post(
url_for('settings.settings_page'),
data={
'llm-model': 'gpt-4o-mini',
'llm-api_key': 'sk-test',
'llm-api_base': '',
'llm-llm_model': 'gpt-4o-mini',
'llm-llm_api_key': 'sk-test',
'llm-llm_api_base': '',
# Attempted injection of token counter fields
'llm-tokens_this_month': '0',
'llm-tokens_total_cumulative': '0',
@@ -471,9 +471,9 @@ def test_cost_fields_are_tamper_proof_via_settings_form(
client.post(
url_for('settings.settings_page'),
data={
'llm-model': 'gpt-4o',
'llm-api_key': 'sk-test',
'llm-api_base': '',
'llm-llm_model': 'gpt-4o',
'llm-llm_api_key': 'sk-test',
'llm-llm_api_base': '',
'llm-cost_usd_this_month': '0', # injection attempt
'llm-cost_usd_total_cumulative': '0', # injection attempt
'application-pager_size': '50',
@@ -1,161 +0,0 @@
#!/usr/bin/env python3
# run from dir above changedetectionio/ dir
# python3 -m unittest changedetectionio.tests.unit.test_llm_settings
import unittest
from pydantic import ValidationError
from changedetectionio.model.LLMSettings import (
LLMSettings,
LLM_DEFAULT_BUDGET_ACTION,
LLM_DEFAULT_LOCAL_TOKEN_MULTIPLIER,
LLM_DEFAULT_MAX_INPUT_CHARS,
LLM_DEFAULT_MAX_SUMMARY_TOKENS,
LLM_DEFAULT_THINKING_BUDGET,
)
class TestLLMSettingsDefaults(unittest.TestCase):
def test_empty_dict_yields_default_model(self):
s = LLMSettings.model_validate({})
self.assertTrue(s.enabled)
self.assertFalse(s.debug)
self.assertEqual(s.model, '')
self.assertEqual(s.api_key, '')
self.assertEqual(s.thinking_budget, LLM_DEFAULT_THINKING_BUDGET)
self.assertEqual(s.max_summary_tokens, LLM_DEFAULT_MAX_SUMMARY_TOKENS)
self.assertEqual(s.local_token_multiplier, LLM_DEFAULT_LOCAL_TOKEN_MULTIPLIER)
self.assertEqual(s.max_input_chars, LLM_DEFAULT_MAX_INPUT_CHARS)
self.assertEqual(s.budget_action, LLM_DEFAULT_BUDGET_ACTION)
self.assertEqual(s.tokens_total_cumulative, 0)
self.assertEqual(s.cost_usd_this_month, 0.0)
def test_default_construct_matches_validate_empty(self):
self.assertEqual(LLMSettings().model_dump(), LLMSettings.model_validate({}).model_dump())
class TestLLMSettingsValidation(unittest.TestCase):
def test_stripped_keys_validate(self):
s = LLMSettings.model_validate({'model': 'gpt-4o-mini', 'enabled': False})
self.assertEqual(s.model, 'gpt-4o-mini')
self.assertFalse(s.enabled)
class TestLLMSettingsTypeCoercion(unittest.TestCase):
def test_select_field_string_int_coerces_to_int(self):
# WTForms SelectField returns the choice key as a string ('500');
# Pydantic coerces to int so storage stays typed.
s = LLMSettings.model_validate({'thinking_budget': '500', 'max_summary_tokens': '5000'})
self.assertEqual(s.thinking_budget, 500)
self.assertEqual(s.max_summary_tokens, 5000)
def test_invalid_int_raises(self):
with self.assertRaises(ValidationError):
LLMSettings.model_validate({'thinking_budget': 'not_a_number'})
class TestLLMSettingsExtraForbid(unittest.TestCase):
def test_unknown_key_raises(self):
# extra='forbid' is the security gate against CWE-915 mass-assignment.
with self.assertRaises(ValidationError) as ctx:
LLMSettings.model_validate({'model': 'gpt-4o-mini', 'evil_field': 'pwn'})
self.assertIn('evil_field', str(ctx.exception))
def test_dunder_key_raises(self):
with self.assertRaises(ValidationError):
LLMSettings.model_validate({'model': 'gpt-4o-mini', '__class__': 'attack'})
def test_legitimate_unknown_key_also_raises(self):
# No "future-tolerant" silent acceptance — new fields must be declared.
with self.assertRaises(ValidationError):
LLMSettings.model_validate({'maybe_future_counter': 42})
def test_legacy_prefixed_key_raises(self):
# Pre-update_31 storage used flat application.llm_* keys (handled by the
# migration). After migration the prefix is gone — and any code path that
# still tries to write a prefixed key into the LLM dict must be rejected
# so the prefix can never reappear through any side channel.
with self.assertRaises(ValidationError):
LLMSettings.model_validate({'llm_model': 'gpt-4o-mini'})
class TestLLMSettingsDumpShapes(unittest.TestCase):
def test_dump_uses_field_names(self):
s = LLMSettings.model_validate({'model': 'gpt-4o-mini'})
out = s.model_dump()
self.assertEqual(out['model'], 'gpt-4o-mini')
self.assertNotIn('llm_model', out)
def test_dump_exclude_connection_drops_provider_fields(self):
s = LLMSettings.model_validate({
'model': 'gpt-4o-mini', 'api_key': 'sk-test', 'api_base': 'https://example',
'provider_kind': 'ollama', 'local_token_multiplier': 5,
'enabled': False, 'tokens_this_month': 42,
})
out = s.model_dump(exclude=set(LLMSettings.CONNECTION_FIELDS))
for k in LLMSettings.CONNECTION_FIELDS:
self.assertNotIn(k, out, f"connection field {k} should be excluded")
# Non-connection fields survive
self.assertFalse(out['enabled'])
self.assertEqual(out['tokens_this_month'], 42)
class TestLLMSettingsFieldGroups(unittest.TestCase):
def test_connection_fields_all_declared(self):
declared = set(LLMSettings.model_fields)
for name in LLMSettings.CONNECTION_FIELDS:
self.assertIn(name, declared, f"CONNECTION_FIELDS lists undeclared field: {name}")
def test_protected_fields_all_declared(self):
declared = set(LLMSettings.model_fields)
for name in LLMSettings.PROTECTED_FIELDS:
self.assertIn(name, declared, f"PROTECTED_FIELDS lists undeclared field: {name}")
def test_connection_and_protected_disjoint(self):
# System-managed counters and user-set provider config must not overlap —
# otherwise a "clear credentials" action would also wipe counters.
overlap = set(LLMSettings.CONNECTION_FIELDS) & set(LLMSettings.PROTECTED_FIELDS)
self.assertEqual(overlap, set(), f"CONNECTION/PROTECTED overlap: {overlap}")
class TestLLMSettingsRoundTrip(unittest.TestCase):
def test_counter_round_trip_via_dump_load(self):
original = LLMSettings.model_validate({
'model': 'gpt-4o-mini',
'tokens_total_cumulative': 123456,
'tokens_this_month': 789,
'tokens_month_key': '2026-05',
'cost_usd_total_cumulative': 12.34,
'cost_usd_this_month': 0.56,
})
roundtripped = LLMSettings.model_validate(original.model_dump())
self.assertEqual(roundtripped.tokens_total_cumulative, 123456)
self.assertEqual(roundtripped.tokens_this_month, 789)
self.assertEqual(roundtripped.tokens_month_key, '2026-05')
self.assertEqual(roundtripped.cost_usd_total_cumulative, 12.34)
self.assertEqual(roundtripped.cost_usd_this_month, 0.56)
def test_form_merge_preserves_counters(self):
# The POST handler pattern: validate existing storage, overlay form input
# (with PROTECTED_FIELDS stripped), re-validate. Counters in storage must
# survive even if the form somehow tried to set them.
existing = LLMSettings.model_validate({
'model': 'gpt-4o-mini', 'tokens_total_cumulative': 99999,
})
form_input = {
'model': 'claude-3-5-haiku-20251001',
'enabled': False,
}
# Strip protected fields from form input as the route handler does
for protected in LLMSettings.PROTECTED_FIELDS:
form_input.pop(protected, None)
merged = LLMSettings.model_validate({**existing.model_dump(), **form_input})
self.assertEqual(merged.model, 'claude-3-5-haiku-20251001')
self.assertFalse(merged.enabled)
self.assertEqual(merged.tokens_total_cumulative, 99999)
if __name__ == '__main__':
unittest.main()
@@ -842,10 +842,6 @@ msgstr ""
msgid "AI Provider"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Master switch — when off, all AI lookups are skipped even if a provider is configured below."
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Third-party data transfer — please read"
msgstr ""
@@ -1096,12 +1092,6 @@ msgstr ""
msgid "characters — currently enforcing: %(limit)s"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid ""
"tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = "
"unlimited)"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "No AI usage recorded yet."
msgstr ""
@@ -3172,8 +3162,12 @@ msgstr ""
msgid "Default AI Change Summary prompt"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html changedetectionio/forms.py
msgid "Max tokens per watch per period"
#: changedetectionio/forms.py
msgid "Max tokens per check"
msgstr ""
#: changedetectionio/forms.py
msgid "Max cumulative tokens (per watch)"
msgstr ""
#: changedetectionio/forms.py
@@ -3184,10 +3178,6 @@ msgstr "Měsíční rozpočet tokenů"
msgid "Max input characters"
msgstr ""
#: changedetectionio/forms.py
msgid "Enable AI / LLM features"
msgstr ""
#: changedetectionio/forms.py
msgid "Replace {{diff}} notification token with AI summary"
msgstr ""
@@ -858,10 +858,6 @@ msgstr ""
msgid "AI Provider"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Master switch — when off, all AI lookups are skipped even if a provider is configured below."
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Third-party data transfer — please read"
msgstr ""
@@ -1112,12 +1108,6 @@ msgstr ""
msgid "characters — currently enforcing: %(limit)s"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid ""
"tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = "
"unlimited)"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "No AI usage recorded yet."
msgstr ""
@@ -3224,8 +3214,12 @@ msgstr ""
msgid "Default AI Change Summary prompt"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html changedetectionio/forms.py
msgid "Max tokens per watch per period"
#: changedetectionio/forms.py
msgid "Max tokens per check"
msgstr ""
#: changedetectionio/forms.py
msgid "Max cumulative tokens (per watch)"
msgstr ""
#: changedetectionio/forms.py
@@ -3236,10 +3230,6 @@ msgstr ""
msgid "Max input characters"
msgstr ""
#: changedetectionio/forms.py
msgid "Enable AI / LLM features"
msgstr ""
#: changedetectionio/forms.py
msgid "Replace {{diff}} notification token with AI summary"
msgstr ""
@@ -840,10 +840,6 @@ msgstr ""
msgid "AI Provider"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Master switch — when off, all AI lookups are skipped even if a provider is configured below."
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Third-party data transfer — please read"
msgstr ""
@@ -1094,12 +1090,6 @@ msgstr ""
msgid "characters — currently enforcing: %(limit)s"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid ""
"tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = "
"unlimited)"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "No AI usage recorded yet."
msgstr ""
@@ -3166,8 +3156,12 @@ msgstr ""
msgid "Default AI Change Summary prompt"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html changedetectionio/forms.py
msgid "Max tokens per watch per period"
#: changedetectionio/forms.py
msgid "Max tokens per check"
msgstr ""
#: changedetectionio/forms.py
msgid "Max cumulative tokens (per watch)"
msgstr ""
#: changedetectionio/forms.py
@@ -3178,10 +3172,6 @@ msgstr ""
msgid "Max input characters"
msgstr ""
#: changedetectionio/forms.py
msgid "Enable AI / LLM features"
msgstr ""
#: changedetectionio/forms.py
msgid "Replace {{diff}} notification token with AI summary"
msgstr ""
@@ -840,10 +840,6 @@ msgstr ""
msgid "AI Provider"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Master switch — when off, all AI lookups are skipped even if a provider is configured below."
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Third-party data transfer — please read"
msgstr ""
@@ -1094,12 +1090,6 @@ msgstr ""
msgid "characters — currently enforcing: %(limit)s"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid ""
"tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = "
"unlimited)"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "No AI usage recorded yet."
msgstr ""
@@ -3166,8 +3156,12 @@ msgstr ""
msgid "Default AI Change Summary prompt"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html changedetectionio/forms.py
msgid "Max tokens per watch per period"
#: changedetectionio/forms.py
msgid "Max tokens per check"
msgstr ""
#: changedetectionio/forms.py
msgid "Max cumulative tokens (per watch)"
msgstr ""
#: changedetectionio/forms.py
@@ -3178,10 +3172,6 @@ msgstr ""
msgid "Max input characters"
msgstr ""
#: changedetectionio/forms.py
msgid "Enable AI / LLM features"
msgstr ""
#: changedetectionio/forms.py
msgid "Replace {{diff}} notification token with AI summary"
msgstr ""
@@ -878,10 +878,6 @@ msgstr ""
msgid "AI Provider"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Master switch — when off, all AI lookups are skipped even if a provider is configured below."
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Third-party data transfer — please read"
msgstr ""
@@ -1132,12 +1128,6 @@ msgstr ""
msgid "characters — currently enforcing: %(limit)s"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid ""
"tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = "
"unlimited)"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "No AI usage recorded yet."
msgstr ""
@@ -2324,11 +2314,11 @@ msgstr "Último Comprobado"
#: changedetectionio/blueprint/watchlist/templates/watch-overview.html
msgid "Changed"
msgstr "Cambiado"
msgstr "Cambiadp"
#: changedetectionio/blueprint/watchlist/templates/watch-overview.html
msgid "Last Changed"
msgstr "Último Cambiado"
msgstr "Último Cambiadp"
#: changedetectionio/blueprint/watchlist/templates/watch-overview.html
msgid "No web page change detection watches configured, please add a URL in the box above, or"
@@ -3239,8 +3229,12 @@ msgstr ""
msgid "Default AI Change Summary prompt"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html changedetectionio/forms.py
msgid "Max tokens per watch per period"
#: changedetectionio/forms.py
msgid "Max tokens per check"
msgstr ""
#: changedetectionio/forms.py
msgid "Max cumulative tokens (per watch)"
msgstr ""
#: changedetectionio/forms.py
@@ -3251,10 +3245,6 @@ msgstr ""
msgid "Max input characters"
msgstr ""
#: changedetectionio/forms.py
msgid "Enable AI / LLM features"
msgstr ""
#: changedetectionio/forms.py
msgid "Replace {{diff}} notification token with AI summary"
msgstr ""
@@ -846,10 +846,6 @@ msgstr ""
msgid "AI Provider"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Master switch — when off, all AI lookups are skipped even if a provider is configured below."
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Third-party data transfer — please read"
msgstr ""
@@ -1100,12 +1096,6 @@ msgstr ""
msgid "characters — currently enforcing: %(limit)s"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid ""
"tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = "
"unlimited)"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "No AI usage recorded yet."
msgstr ""
@@ -3179,8 +3169,12 @@ msgstr ""
msgid "Default AI Change Summary prompt"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html changedetectionio/forms.py
msgid "Max tokens per watch per period"
#: changedetectionio/forms.py
msgid "Max tokens per check"
msgstr ""
#: changedetectionio/forms.py
msgid "Max cumulative tokens (per watch)"
msgstr ""
#: changedetectionio/forms.py
@@ -3191,10 +3185,6 @@ msgstr ""
msgid "Max input characters"
msgstr ""
#: changedetectionio/forms.py
msgid "Enable AI / LLM features"
msgstr ""
#: changedetectionio/forms.py
msgid "Replace {{diff}} notification token with AI summary"
msgstr ""
@@ -842,10 +842,6 @@ msgstr ""
msgid "AI Provider"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Master switch — when off, all AI lookups are skipped even if a provider is configured below."
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Third-party data transfer — please read"
msgstr ""
@@ -1096,12 +1092,6 @@ msgstr ""
msgid "characters — currently enforcing: %(limit)s"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid ""
"tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = "
"unlimited)"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "No AI usage recorded yet."
msgstr ""
@@ -3168,8 +3158,12 @@ msgstr ""
msgid "Default AI Change Summary prompt"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html changedetectionio/forms.py
msgid "Max tokens per watch per period"
#: changedetectionio/forms.py
msgid "Max tokens per check"
msgstr ""
#: changedetectionio/forms.py
msgid "Max cumulative tokens (per watch)"
msgstr ""
#: changedetectionio/forms.py
@@ -3180,10 +3174,6 @@ msgstr ""
msgid "Max input characters"
msgstr ""
#: changedetectionio/forms.py
msgid "Enable AI / LLM features"
msgstr ""
#: changedetectionio/forms.py
msgid "Replace {{diff}} notification token with AI summary"
msgstr ""
@@ -847,10 +847,6 @@ msgstr ""
msgid "AI Provider"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Master switch — when off, all AI lookups are skipped even if a provider is configured below."
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Third-party data transfer — please read"
msgstr ""
@@ -1101,12 +1097,6 @@ msgstr ""
msgid "characters — currently enforcing: %(limit)s"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid ""
"tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = "
"unlimited)"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "No AI usage recorded yet."
msgstr ""
@@ -3185,8 +3175,12 @@ msgstr ""
msgid "Default AI Change Summary prompt"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html changedetectionio/forms.py
msgid "Max tokens per watch per period"
#: changedetectionio/forms.py
msgid "Max tokens per check"
msgstr ""
#: changedetectionio/forms.py
msgid "Max cumulative tokens (per watch)"
msgstr ""
#: changedetectionio/forms.py
@@ -3197,10 +3191,6 @@ msgstr ""
msgid "Max input characters"
msgstr ""
#: changedetectionio/forms.py
msgid "Enable AI / LLM features"
msgstr ""
#: changedetectionio/forms.py
msgid "Replace {{diff}} notification token with AI summary"
msgstr ""
@@ -844,10 +844,6 @@ msgstr "AI 프로바이더 설정"
msgid "AI Provider"
msgstr "AI 프로바이더"
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Master switch — when off, all AI lookups are skipped even if a provider is configured below."
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Third-party data transfer — please read"
msgstr "제3자 데이터 전송 - 읽어 주세요"
@@ -1102,12 +1098,6 @@ msgstr "(<code>LLM_MAX_INPUT_CHARS</code>로 설정됨)"
msgid "characters — currently enforcing: %(limit)s"
msgstr "문자 - 현재 적용 중: %(limit)s"
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid ""
"tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = "
"unlimited)"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "No AI usage recorded yet."
msgstr "아직 기록된 AI 사용량이 없습니다."
@@ -3176,9 +3166,13 @@ msgstr ""
msgid "Default AI Change Summary prompt"
msgstr "기본 AI 변경 요약 프롬프트"
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html changedetectionio/forms.py
msgid "Max tokens per watch per period"
msgstr ""
#: changedetectionio/forms.py
msgid "Max tokens per check"
msgstr "확인당 최대 토큰 수"
#: changedetectionio/forms.py
msgid "Max cumulative tokens (per watch)"
msgstr "최대 누적 토큰 수 (모니터링별)"
#: changedetectionio/forms.py
msgid "Monthly token budget"
@@ -3188,10 +3182,6 @@ msgstr "월간 토큰 예산"
msgid "Max input characters"
msgstr "최대 입력 문자 수"
#: changedetectionio/forms.py
msgid "Enable AI / LLM features"
msgstr ""
#: changedetectionio/forms.py
msgid "Replace {{diff}} notification token with AI summary"
msgstr "{{diff}} 알림 토큰을 AI 요약으로 대체"
+8 -18
View File
@@ -6,9 +6,9 @@
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: changedetection.io 0.55.5\n"
"Project-Id-Version: changedetection.io 0.55.4\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2026-05-25 10:09+0200\n"
"POT-Creation-Date: 2026-05-19 11:38+0200\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
@@ -839,10 +839,6 @@ msgstr ""
msgid "AI Provider"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Master switch — when off, all AI lookups are skipped even if a provider is configured below."
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Third-party data transfer — please read"
msgstr ""
@@ -1093,12 +1089,6 @@ msgstr ""
msgid "characters — currently enforcing: %(limit)s"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid ""
"tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = "
"unlimited)"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "No AI usage recorded yet."
msgstr ""
@@ -3165,8 +3155,12 @@ msgstr ""
msgid "Default AI Change Summary prompt"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html changedetectionio/forms.py
msgid "Max tokens per watch per period"
#: changedetectionio/forms.py
msgid "Max tokens per check"
msgstr ""
#: changedetectionio/forms.py
msgid "Max cumulative tokens (per watch)"
msgstr ""
#: changedetectionio/forms.py
@@ -3177,10 +3171,6 @@ msgstr ""
msgid "Max input characters"
msgstr ""
#: changedetectionio/forms.py
msgid "Enable AI / LLM features"
msgstr ""
#: changedetectionio/forms.py
msgid "Replace {{diff}} notification token with AI summary"
msgstr ""
@@ -865,10 +865,6 @@ msgstr ""
msgid "AI Provider"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Master switch — when off, all AI lookups are skipped even if a provider is configured below."
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Third-party data transfer — please read"
msgstr ""
@@ -1119,12 +1115,6 @@ msgstr ""
msgid "characters — currently enforcing: %(limit)s"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid ""
"tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = "
"unlimited)"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "No AI usage recorded yet."
msgstr ""
@@ -3216,8 +3206,12 @@ msgstr ""
msgid "Default AI Change Summary prompt"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html changedetectionio/forms.py
msgid "Max tokens per watch per period"
#: changedetectionio/forms.py
msgid "Max tokens per check"
msgstr ""
#: changedetectionio/forms.py
msgid "Max cumulative tokens (per watch)"
msgstr ""
#: changedetectionio/forms.py
@@ -3228,10 +3222,6 @@ msgstr ""
msgid "Max input characters"
msgstr ""
#: changedetectionio/forms.py
msgid "Enable AI / LLM features"
msgstr ""
#: changedetectionio/forms.py
msgid "Replace {{diff}} notification token with AI summary"
msgstr ""
@@ -875,10 +875,6 @@ msgstr ""
msgid "AI Provider"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Master switch — when off, all AI lookups are skipped even if a provider is configured below."
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Third-party data transfer — please read"
msgstr ""
@@ -1129,12 +1125,6 @@ msgstr ""
msgid "characters — currently enforcing: %(limit)s"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid ""
"tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = "
"unlimited)"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "No AI usage recorded yet."
msgstr ""
@@ -3219,8 +3209,12 @@ msgstr ""
msgid "Default AI Change Summary prompt"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html changedetectionio/forms.py
msgid "Max tokens per watch per period"
#: changedetectionio/forms.py
msgid "Max tokens per check"
msgstr ""
#: changedetectionio/forms.py
msgid "Max cumulative tokens (per watch)"
msgstr ""
#: changedetectionio/forms.py
@@ -3231,10 +3225,6 @@ msgstr ""
msgid "Max input characters"
msgstr ""
#: changedetectionio/forms.py
msgid "Enable AI / LLM features"
msgstr ""
#: changedetectionio/forms.py
msgid "Replace {{diff}} notification token with AI summary"
msgstr ""
@@ -855,10 +855,6 @@ msgstr ""
msgid "AI Provider"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Master switch — when off, all AI lookups are skipped even if a provider is configured below."
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Third-party data transfer — please read"
msgstr ""
@@ -1109,12 +1105,6 @@ msgstr ""
msgid "characters — currently enforcing: %(limit)s"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid ""
"tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = "
"unlimited)"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "No AI usage recorded yet."
msgstr ""
@@ -3198,8 +3188,12 @@ msgstr ""
msgid "Default AI Change Summary prompt"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html changedetectionio/forms.py
msgid "Max tokens per watch per period"
#: changedetectionio/forms.py
msgid "Max tokens per check"
msgstr ""
#: changedetectionio/forms.py
msgid "Max cumulative tokens (per watch)"
msgstr ""
#: changedetectionio/forms.py
@@ -3210,10 +3204,6 @@ msgstr ""
msgid "Max input characters"
msgstr ""
#: changedetectionio/forms.py
msgid "Enable AI / LLM features"
msgstr ""
#: changedetectionio/forms.py
msgid "Replace {{diff}} notification token with AI summary"
msgstr ""
@@ -844,10 +844,6 @@ msgstr ""
msgid "AI Provider"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Master switch — when off, all AI lookups are skipped even if a provider is configured below."
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Third-party data transfer — please read"
msgstr ""
@@ -1098,12 +1094,6 @@ msgstr ""
msgid "characters — currently enforcing: %(limit)s"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid ""
"tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = "
"unlimited)"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "No AI usage recorded yet."
msgstr ""
@@ -3171,8 +3161,12 @@ msgstr ""
msgid "Default AI Change Summary prompt"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html changedetectionio/forms.py
msgid "Max tokens per watch per period"
#: changedetectionio/forms.py
msgid "Max tokens per check"
msgstr ""
#: changedetectionio/forms.py
msgid "Max cumulative tokens (per watch)"
msgstr ""
#: changedetectionio/forms.py
@@ -3183,10 +3177,6 @@ msgstr ""
msgid "Max input characters"
msgstr ""
#: changedetectionio/forms.py
msgid "Enable AI / LLM features"
msgstr ""
#: changedetectionio/forms.py
msgid "Replace {{diff}} notification token with AI summary"
msgstr ""
@@ -843,10 +843,6 @@ msgstr ""
msgid "AI Provider"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Master switch — when off, all AI lookups are skipped even if a provider is configured below."
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "Third-party data transfer — please read"
msgstr ""
@@ -1097,12 +1093,6 @@ msgstr ""
msgid "characters — currently enforcing: %(limit)s"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid ""
"tokens — skips AI evaluation on a watch once its usage within the current period (monthly) hits this cap (0 = "
"unlimited)"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html
msgid "No AI usage recorded yet."
msgstr ""
@@ -3170,8 +3160,12 @@ msgstr ""
msgid "Default AI Change Summary prompt"
msgstr ""
#: changedetectionio/blueprint/settings/templates/settings_llm_tab.html changedetectionio/forms.py
msgid "Max tokens per watch per period"
#: changedetectionio/forms.py
msgid "Max tokens per check"
msgstr ""
#: changedetectionio/forms.py
msgid "Max cumulative tokens (per watch)"
msgstr ""
#: changedetectionio/forms.py
@@ -3182,10 +3176,6 @@ msgstr ""
msgid "Max input characters"
msgstr ""
#: changedetectionio/forms.py
msgid "Enable AI / LLM features"
msgstr ""
#: changedetectionio/forms.py
msgid "Replace {{diff}} notification token with AI summary"
msgstr ""
+7 -20
View File
@@ -432,15 +432,9 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
update_obj['_llm_result'] = None
update_obj['_llm_intent'] = ''
update_obj['_llm_change_summary'] = ''
# skip_check: when budget exceeded, don't run LLM or the check.
# Also gated on llm_enabled — a disabled LLM can't be spending tokens,
# so the budget enforcement shouldn't suppress changes when the user
# has explicitly switched LLM off.
from changedetectionio.llm.evaluator import is_llm_features_disabled as _is_llm_features_disabled, get_llm_settings as _get_llm_settings
_llm_settings = _get_llm_settings(datastore)
_llm_master_enabled = _llm_settings.enabled and not _is_llm_features_disabled()
_llm_budget_action = _llm_settings.budget_action
if _llm_master_enabled and _llm_budget_action == 'skip_check':
# skip_check: when budget exceeded, don't run LLM or the check
_llm_budget_action = datastore.data['settings']['application'].get('llm_budget_action', 'skip_llm')
if _llm_budget_action == 'skip_check':
from changedetectionio.llm.evaluator import is_global_token_budget_exceeded
if is_global_token_budget_exceeded(datastore):
logger.info(f"LLM monthly budget exceeded — skipping check for {uuid} (budget_action=skip_check)")
@@ -450,14 +444,9 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
try:
from changedetectionio.llm.evaluator import (
evaluate_change, resolve_intent, resolve_llm_field,
summarise_change, _runtime_llm_config,
summarise_change, get_llm_config,
)
# _runtime_llm_config returns None (and logs a debug skip
# message) when the master 'llm_enabled' toggle is off, so
# the whole block — diff computation, status minitext, and
# the two executor dispatches — is skipped, not just the
# inner LLM lookups.
_llm_cfg = _runtime_llm_config(datastore)
_llm_cfg = get_llm_config(datastore)
if _llm_cfg:
# Compute unified diff once — used by both intent and summary
_watch_dates = list(watch.history.keys())
@@ -549,10 +538,8 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
get_effective_summary_prompt, build_summary_cache_prompt,
)
_llm_to_version = list(watch.history.keys())[-1]
from changedetectionio.llm.evaluator import get_llm_settings as _get_llm_settings_inner
_ls = _get_llm_settings_inner(datastore)
_llm_max_summary_tokens = _ls.max_summary_tokens
_llm_model = _ls.model
_llm_max_summary_tokens = datastore.data['settings']['application'].get('llm_max_summary_tokens', 3000)
_llm_model = (datastore.data['settings']['application'].get('llm') or {}).get('model', '')
_llm_cache_prompt = build_summary_cache_prompt(
effective_prompt=get_effective_summary_prompt(watch, datastore),
max_summary_tokens=_llm_max_summary_tokens,
-8
View File
@@ -621,14 +621,6 @@ components:
type: [integer, 'null']
readOnly: true
description: Total tokens consumed by the AI across all checks for this watch.
llm_tokens_this_period:
type: [integer, 'null']
readOnly: true
description: Tokens consumed by the AI on this watch within the current rollover period (currently month). Used to enforce max_tokens_per_count_period.
llm_tokens_period_key:
type: [string, 'null']
readOnly: true
description: Identifier of the current rollover period (e.g. "2026-05"). Set automatically; resets llm_tokens_this_period when the period changes.
DaySchedule:
type: object
-3
View File
@@ -148,9 +148,6 @@ pluggy ~= 1.6
# LLM intent-based change evaluation (multi-provider via litellm)
litellm>=1.40.0,<1.83.1 # 1.83.11.83.14 exact-pin jsonschema==4.23.0, conflicting with openapi-spec-validator's >=4.24.0 floor; re-evaluate when litellm fixes this
# Used today for LLMSettings (model/LLMSettings.py); transitively pulled by litellm but pinned explicitly
# so the validation/typing layer doesn't disappear if litellm drops it.
pydantic>=2.0,<3.0
# BM25 relevance trimming for large snapshots (pure Python, no ML)
rank-bm25>=0.2.2