Adding missing file

changedetectionio/PLUGIN_README.md

@@ -38,6 +38,66 @@ def ui_edit_stats_extras(watch):
 
 3. The HTML you return will be included in the Stats tab.
 
+## LLM Query Hooks
+
+External packages can observe and modify every LiteLLM call (intent evaluation,
+change summaries, restock extraction, connection tests, etc.).
+
+### `llm_query_alter` — before the request
+
+Return a dict of keys to merge into the call context (`messages`, `model`,
+`max_tokens`, `api_key`, `api_base`, `extra_body`, …).
+
+```python
+from changedetectionio.pluggy_interface import hookimpl
+
+@hookimpl
+def llm_query_alter(llm_context):
+    # llm_context includes:
+    #   purpose, watch, datastore, app_guid, watch_uuid, timestamp_utc,
+    #   settings (full application settings copy), model, messages, ...
+    if llm_context.get('purpose') != 'evaluate_change':
+        return None
+    messages = list(llm_context['messages'])
+    messages.append({'role': 'user', 'content': 'Extra auditing instruction.'})
+    return {'messages': messages}
+```
+
+### `llm_query_finalize` — after success or failure
+
+Use for token/cost accounting (MySQL, Prometheus, billing exports, etc.).
+
+```python
+@hookimpl
+def llm_query_finalize(llm_context, result, error):
+    if error:
+        log_failure(llm_context['app_guid'], llm_context['watch_uuid'], error)
+        return
+    # result keys: text, total_tokens, input_tokens, output_tokens,
+    #   cost_usd, litellm_response_cost_usd, model, finish_reason, duration_seconds
+    record_usage(
+        app_guid=llm_context['app_guid'],
+        watch_uuid=llm_context['watch_uuid'],
+        purpose=llm_context['purpose'],
+        tokens=result['total_tokens'],
+        cost_usd=result['cost_usd'],
+        at=llm_context['timestamp_utc'],
+    )
+```
+
+Register via setuptools entry point (namespace `changedetectionio`), same as other plugins:
+
+```python
+entry_points={
+    'changedetectionio': [
+        'llm_accounting = my_package.llm_plugin',
+    ],
+},
+```
+
+**Purpose values** (call-site identifiers): `evaluate_change`, `summarise_change`,
+`run_setup`, `preview_extract`, `restock_extract`, `connection_test`.
+
 ## Plugin Loading
 
 Plugins can be loaded from:

changedetectionio/blueprint/settings/__init__.py

@@ -14,8 +14,10 @@ from changedetectionio.auth_decorator import login_optionally_required
 
 def construct_blueprint(datastore: ChangeDetectionStore):
     from changedetectionio.blueprint.settings.llm import construct_llm_blueprint
+    from changedetectionio.llm.evaluator import is_llm_features_disabled
     settings_blueprint = Blueprint('settings', __name__, template_folder="templates")
-    settings_blueprint.register_blueprint(construct_llm_blueprint(datastore), url_prefix='/llm')
+    if not is_llm_features_disabled():
+        settings_blueprint.register_blueprint(construct_llm_blueprint(datastore), url_prefix='/llm')
 
     @settings_blueprint.route("", methods=['GET', "POST"])
     @login_optionally_required

changedetectionio/blueprint/settings/llm.py

@@ -134,7 +134,7 @@ def construct_llm_blueprint(datastore: ChangeDetectionStore):
     @login_optionally_required
     def llm_test():
         from flask import request
-        from changedetectionio.llm.client import completion
+        from changedetectionio.llm.invocation import llm_completion
         from changedetectionio.validate_url import is_llm_api_base_safe
 
         # Pull stored config as the fallback, then override with anything the
@@ -194,7 +194,10 @@ def construct_llm_blueprint(datastore: ChangeDetectionStore):
             # cloud reasoning models (e.g. ollama.com hosting qwen3.5:397b takes ~60s on
             # first hit) even though the same call succeeds in production.
             from changedetectionio.llm.evaluator import apply_local_token_multiplier
-            text, total_tokens, input_tokens, output_tokens = completion(
+            text, total_tokens, input_tokens, output_tokens = llm_completion(
+                'connection_test',
+                watch=None,
+                datastore=datastore,
                 model=model,
                 messages=[{'role': 'user', 'content':
                     'Respond with just the word: ready'}],

changedetectionio/blueprint/settings/templates/settings.html

@@ -34,7 +34,9 @@
             <li class="tab"><a href="#plugin-{{ tab.plugin_id }}">{{ tab.tab_label }}</a></li>
                 {% endfor %}
             {% endif %}
+            {% if not llm_features_disabled %}
             <li class="tab"><a href="#ai">{{ _('AI / LLM') }}</a></li>
+            {% endif %}
             <li class="tab"><a href="#info">{{ _('Info') }}</a></li>
         </ul>
     </div>
@@ -394,7 +396,9 @@ nav
             </div>
                 {% endfor %}
             {% endif %}
+            {% if not llm_features_disabled %}
             {% include 'settings_llm_tab.html' %}
+            {% endif %}
             <div class="tab-pane-inner" id="info">
                 <p><strong>{{ _('Uptime:') }}</strong> {{ uptime_seconds|format_duration }}</p>
                 <p><strong>{{ _('Python version:') }}</strong> {{ python_version }}</p>

changedetectionio/blueprint/ui/templates/edit.html

@@ -57,7 +57,9 @@
             {% if capabilities.supports_visual_selector %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">{{ _('Visual Filter Selector') }}</a></li>
             {% endif %}
+            {% if not llm_features_disabled %}
             <li class="tab"><a href="#ai-llm">{{ _('AI / LLM') }}</a></li>
+            {% endif %}
             {% if capabilities.supports_text_filters_and_triggers %}
             <li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">{{ _('Filters & Triggers') }}</a></li>
             <li class="tab" id="conditions-tab"><a href="#conditions">{{ _('Conditions') }}</a></li>
@@ -321,9 +323,11 @@ Math: {{ 1 + 1 }}") }}
                     </div>
                 </div>
             </div>
+            {% if not llm_features_disabled %}
             <div class="tab-pane-inner" id="ai-llm">
                 {% include "edit/include_llm_intent.html" %}
             </div>
+            {% endif %}
             <div class="tab-pane-inner" id="filters-and-triggers">
 
                 <span id="activate-text-preview" class="pure-button pure-button-primary button-xsmall">{{ _('Activate preview') }}</span>
@@ -503,7 +507,7 @@ Math: {{ 1 + 1 }}") }}
                             <td>{{ _('Server type reply') }}</td>
                             <td>{{ watch.get('remote_server_reply') }}</td>
                         </tr>
-                        {% if settings_application.get('llm', {}).get('model') %}
+                        {% if not llm_features_disabled and settings_application.get('llm', {}).get('model') %}
                         <tr>
                             <td>{{ _('AI tokens (last check)') }}</td>
                             <td>{{ "{:,}".format(watch.get('llm_last_tokens_used') or 0) }}</td>

changedetectionio/flask_app.py

@@ -522,6 +522,11 @@ def changedetection_app(config=None, datastore_o=None):
         available_languages=available_languages
     )
 
+    @app.context_processor
+    def inject_llm_features_disabled():
+        from changedetectionio.llm.evaluator import is_llm_features_disabled
+        return dict(llm_features_disabled=is_llm_features_disabled())
+
     # Set up a request hook to check authentication for all routes
     @app.before_request
     def check_authentication():

changedetectionio/llm/client.py

@@ -54,12 +54,26 @@ def _install_litellm_debug():
     logger.info("LLM client: litellm debug logging routed through loguru")
 
 
+def _litellm_response_cost_usd(response) -> float | None:
+    """Extract provider/litellm-reported cost from a completion response, if present."""
+    try:
+        from litellm.cost_calculator import get_response_cost_from_hidden_params
+        hidden = getattr(response, '_hidden_params', None) or {}
+        cost = get_response_cost_from_hidden_params(hidden)
+        if cost is not None:
+            return float(cost)
+    except Exception:
+        pass
+    return None
+
+
 def completion(model: str, messages: list, api_key: str = None,
                api_base: str = None, timeout: int = DEFAULT_TIMEOUT,
                max_tokens: int = None, extra_body: dict = None,
-               debug: bool = False) -> tuple[str, int, int, int]:
+               debug: bool = False, return_metadata: bool = False):
     """
     Call the LLM and return (response_text, total_tokens, input_tokens, output_tokens).
+    When return_metadata=True, appends a dict with finish_reason and litellm cost fields.
     Retries up to DEFAULT_RETRIES times on timeout or connection errors.
     Token counts are 0 if the provider doesn't return usage data.
     Raises on network/auth errors — callers handle gracefully.
@@ -134,6 +148,12 @@ def completion(model: str, messages: list, api_key: str = None,
                 f"tokens={total_tokens} (in={input_tokens} out={output_tokens}) "
                 f"text_len={len(text)}"
             )
+            if return_metadata:
+                metadata = {'finish_reason': finish}
+                litellm_cost = _litellm_response_cost_usd(response)
+                if litellm_cost is not None:
+                    metadata['litellm_response_cost_usd'] = litellm_cost
+                return text, total_tokens, input_tokens, output_tokens, metadata
             return text, total_tokens, input_tokens, output_tokens
 
         except _retryable as e:

changedetectionio/llm/evaluator.py

@@ -20,7 +20,9 @@ from dataclasses import dataclass
 from datetime import datetime, timezone
 from loguru import logger
 
-from . import client as llm_client
+from changedetectionio.strtobool import strtobool
+
+from .invocation import llm_completion
 from .prompt_builder import (
     build_change_summary_prompt, build_change_summary_system_prompt,
     build_eval_prompt, build_eval_system_prompt,
@@ -31,6 +33,11 @@ from .response_parser import parse_eval_response, parse_preview_response, parse_
 
 _DEFAULT_MAX_INPUT_CHARS = 100_000
 
+
+def is_llm_features_disabled() -> bool:
+    """True when the LLM_FEATURES_DISABLED env var is set to a truthy value."""
+    return bool(strtobool(os.getenv('LLM_FEATURES_DISABLED', '')))
+
 def _get_max_input_chars(datastore) -> int:
     """Max input characters to send to the LLM. Resolution: env var → datastore → 100,000.
     Always returns at least 1 — unlimited is not permitted.
@@ -207,6 +214,8 @@ def get_llm_config(datastore) -> dict | None:
       1. Environment variables: LLM_MODEL, LLM_API_KEY, LLM_API_BASE
       2. Datastore settings (set via UI)
     """
+    if is_llm_features_disabled():
+        return None
     # 1. Environment variable override
     env_model = os.getenv('LLM_MODEL', '').strip()
     if env_model:
@@ -225,6 +234,8 @@ def get_llm_config(datastore) -> dict | None:
 
 def llm_configured_via_env() -> bool:
     """True when LLM config comes from environment variables, not the UI."""
+    if is_llm_features_disabled():
+        return False
     return bool(os.getenv('LLM_MODEL', '').strip())
 
 
@@ -414,7 +425,10 @@ def run_setup(watch, datastore, snapshot_text: str) -> None:
     user_prompt = build_setup_prompt(intent, snapshot_text, url=url)
 
     try:
-        raw, tokens, *_ = llm_client.completion(
+        raw, tokens, *_ = llm_completion(
+            'run_setup',
+            watch=watch,
+            datastore=datastore,
             model=cfg['model'],
             messages=[
                 _cached_system(system_prompt, model=cfg['model']),
@@ -566,7 +580,10 @@ def summarise_change(watch, datastore, diff: str, current_snapshot: str = '') ->
     _extra_body = _thinking_extra_body(cfg['model'], _thinking_budget)
 
     try:
-        _resp = llm_client.completion(
+        _resp = llm_completion(
+            'summarise_change',
+            watch=watch,
+            datastore=datastore,
             model=cfg['model'],
             messages=[
                 _cached_system(system_prompt, model=cfg['model']),
@@ -635,7 +652,10 @@ def preview_extract(watch, datastore, content: str) -> dict | None:
     user_prompt = build_preview_prompt(intent, content, url=url, title=title)
 
     try:
-        raw, tokens, *_ = llm_client.completion(
+        raw, tokens, *_ = llm_completion(
+            'preview_extract',
+            watch=watch,
+            datastore=datastore,
             model=cfg['model'],
             messages=[
                 _cached_system(system_prompt, model=cfg['model']),
@@ -720,7 +740,10 @@ def evaluate_change(watch, datastore, diff: str, current_snapshot: str = '') ->
     )
 
     try:
-        _resp = llm_client.completion(
+        _resp = llm_completion(
+            'evaluate_change',
+            watch=watch,
+            datastore=datastore,
             model=cfg['model'],
             messages=[
                 _cached_system(system_prompt, model=cfg['model']),

changedetectionio/llm/invocation.py

@@ -0,0 +1,151 @@
+"""
+Central LLM invocation path with pluggy hooks.
+
+All production litellm calls should go through llm_completion() so external plugins
+can alter requests (llm_query_alter) and record usage afterward (llm_query_finalize).
+"""
+
+import time
+from copy import deepcopy
+from datetime import datetime, timezone
+
+from loguru import logger
+
+from changedetectionio.pluggy_interface import apply_llm_query_alter, apply_llm_query_finalize
+
+from . import client as llm_client
+
+
+def build_llm_context(
+    purpose: str,
+    *,
+    watch=None,
+    datastore=None,
+    model: str,
+    messages: list,
+    api_key: str = None,
+    api_base: str = None,
+    timeout: int = None,
+    max_tokens: int = None,
+    extra_body: dict = None,
+    debug: bool = False,
+) -> dict:
+    """Build the context dict for llm_query_alter / llm_query_finalize.
+
+    See ChangeDetectionSpec.llm_query_finalize in pluggy_interface.py for the
+    full field reference (purpose, app_guid, watch_uuid, settings, result keys, …).
+    """
+    app_guid = None
+    settings = None
+    if datastore is not None:
+        try:
+            app_guid = datastore.data.get('app_guid')
+            settings = deepcopy(datastore.data.get('settings') or {})
+        except Exception:
+            pass
+
+    watch_uuid = None
+    if watch is not None:
+        watch_uuid = watch.get('uuid') if isinstance(watch, dict) else getattr(watch, 'uuid', None)
+
+    return {
+        'purpose': purpose,
+        'watch': watch,
+        'datastore': datastore,
+        'app_guid': app_guid,
+        'watch_uuid': watch_uuid,
+        'timestamp_utc': datetime.now(timezone.utc).isoformat(),
+        'settings': settings,
+        'model': model,
+        'messages': messages,
+        'api_key': api_key,
+        'api_base': api_base,
+        'timeout': timeout,
+        'max_tokens': max_tokens,
+        'extra_body': extra_body,
+        'debug': debug,
+    }
+
+
+def _completion_cost_usd(model: str, input_tokens: int, output_tokens: int, metadata: dict) -> float:
+    """Prefer litellm's response cost when present, else use the app's pricing estimate."""
+    litellm_cost = (metadata or {}).get('litellm_response_cost_usd')
+    if litellm_cost is not None:
+        try:
+            return float(litellm_cost)
+        except (TypeError, ValueError):
+            pass
+    from changedetectionio.llm.evaluator import _estimate_cost_usd
+    return _estimate_cost_usd(model, input_tokens, output_tokens)
+
+
+def llm_completion(
+    purpose: str,
+    *,
+    watch=None,
+    datastore=None,
+    model: str,
+    messages: list,
+    api_key: str = None,
+    api_base: str = None,
+    timeout: int = None,
+    max_tokens: int = None,
+    extra_body: dict = None,
+    debug: bool = False,
+) -> tuple[str, int, int, int]:
+    """
+    Run litellm.completion with pluggy alter/finalize hooks.
+
+    Returns (response_text, total_tokens, input_tokens, output_tokens) — same as
+    llm.client.completion for drop-in replacement at call sites.
+    """
+    llm_context = build_llm_context(
+        purpose,
+        watch=watch,
+        datastore=datastore,
+        model=model,
+        messages=messages,
+        api_key=api_key,
+        api_base=api_base,
+        timeout=timeout,
+        max_tokens=max_tokens,
+        extra_body=extra_body,
+        debug=debug,
+    )
+    llm_context = apply_llm_query_alter(llm_context)
+
+    started = time.monotonic()
+    result = None
+    error = None
+    try:
+        text, total_tokens, input_tokens, output_tokens, metadata = llm_client.completion(
+            model=llm_context['model'],
+            messages=llm_context['messages'],
+            api_key=llm_context.get('api_key'),
+            api_base=llm_context.get('api_base'),
+            timeout=llm_context.get('timeout'),
+            max_tokens=llm_context.get('max_tokens'),
+            extra_body=llm_context.get('extra_body'),
+            debug=bool(llm_context.get('debug')),
+            return_metadata=True,
+        )
+        cost_usd = _completion_cost_usd(
+            llm_context['model'], input_tokens, output_tokens, metadata,
+        )
+        result = {
+            'text': text,
+            'total_tokens': total_tokens,
+            'input_tokens': input_tokens,
+            'output_tokens': output_tokens,
+            'cost_usd': cost_usd,
+            'litellm_response_cost_usd': (metadata or {}).get('litellm_response_cost_usd'),
+            'model': llm_context['model'],
+            'finish_reason': (metadata or {}).get('finish_reason'),
+            'duration_seconds': time.monotonic() - started,
+        }
+        return text, total_tokens, input_tokens, output_tokens
+    except Exception as e:
+        error = e
+        raise
+    finally:
+        apply_llm_query_finalize(llm_context, result, error)

changedetectionio/pluggy_interface.py

@@ -175,6 +175,75 @@ class ChangeDetectionSpec:
         """
         pass
 
+    @hookspec
+    def llm_query_alter(llm_context):
+        """Modify an LLM request before litellm.completion is called.
+
+        Called for every LLM invocation (intent evaluation, change summaries,
+        restock extraction, connection tests, etc.). Plugins can adjust messages,
+        model, max_tokens, or other completion kwargs.
+
+        Args:
+            llm_context: dict describing the call. Common keys:
+                purpose (str): call-site id, e.g. 'evaluate_change', 'summarise_change'
+                watch (dict|None): watch being processed, when applicable
+                datastore: ChangeDetectionStore instance, when available
+                app_guid (str|None): application GUID from datastore
+                watch_uuid (str|None): watch UUID
+                timestamp_utc (str): ISO-8601 UTC time when the call started
+                settings (dict): copy of datastore.data['settings'] when datastore set
+                model, messages, api_key, api_base, timeout, max_tokens, extra_body, debug
+
+        Returns:
+            dict or None: Keys to merge into llm_context (later plugins see merged state).
+                          Return None to leave the context unchanged.
+        """
+        pass
+
+    @hookspec
+    def llm_query_finalize(llm_context, result, error):
+        """Called after each litellm.completion attempt finishes (success or failure).
+
+        Use for external accounting (MySQL, Prometheus, billing exports, etc.).
+
+        Args:
+            llm_context: dict describing the call (same object passed to llm_query_alter,
+                         after any plugin merges). Keys always present when built by the app:
+
+                purpose (str): call-site id — one of:
+                    'evaluate_change', 'summarise_change', 'run_setup',
+                    'preview_extract', 'restock_extract', 'connection_test'
+                app_guid (str|None): stable application GUID (datastore.data['app_guid'])
+                watch_uuid (str|None): watch UUID, or None when no watch (e.g. connection test)
+                timestamp_utc (str): ISO-8601 UTC time when the request started
+                settings (dict|None): deep copy of datastore.data['settings'] (application,
+                                      tags, notification profiles, llm config, etc.)
+                watch (dict|None): watch dict under processing, when applicable
+                datastore: ChangeDetectionStore instance, when available
+                model (str): model string sent to litellm (after alter hooks)
+                messages (list): chat messages sent to litellm (after alter hooks)
+                api_key, api_base, timeout, max_tokens, extra_body, debug: completion kwargs
+
+            result: dict on success, None on failure:
+                {
+                    'text': str,                      # model response body
+                    'total_tokens': int,
+                    'input_tokens': int,
+                    'output_tokens': int,
+                    'cost_usd': float,                # litellm response cost if reported,
+                                                      # else litellm cost_per_token estimate
+                    'litellm_response_cost_usd': float|None,  # provider-reported only
+                    'model': str,
+                    'finish_reason': str|None,        # e.g. 'stop', 'length'
+                    'duration_seconds': float,        # wall time for the completion call
+                }
+            error: Exception instance if the call failed, else None
+
+        Returns:
+            None
+        """
+        pass
+
     @hookspec
     def get_html_head_extras():
         """Return HTML to inject into the <head> of every page via base.html.
@@ -691,6 +760,47 @@ def apply_update_finalize(update_handler, watch, datastore, processing_exception
         logger.exception(f"update_finalize hook exception details:")
 
 
+_LLM_CONTEXT_KEYS = frozenset({
+    'model', 'messages', 'api_key', 'api_base', 'timeout', 'max_tokens', 'extra_body', 'debug',
+})
+
+
+def apply_llm_query_alter(llm_context: dict) -> dict:
+    """Apply llm_query_alter hooks; merge plugin overrides into the call context."""
+    current = dict(llm_context)
+    try:
+        results = plugin_manager.hook.llm_query_alter(llm_context=current)
+    except Exception as e:
+        logger.error(f"Error in llm_query_alter hook: {e}")
+        logger.exception("llm_query_alter hook exception details:")
+        return current
+
+    if results:
+        for result in results:
+            if result and isinstance(result, dict):
+                for key, value in result.items():
+                    if key in _LLM_CONTEXT_KEYS or key in current:
+                        current[key] = value
+                logger.debug(
+                    f"LLM query altered by plugin (purpose={current.get('purpose')!r} "
+                    f"watch={current.get('watch_uuid')!r})"
+                )
+    return current
+
+
+def apply_llm_query_finalize(llm_context: dict, result: dict | None, error: Exception | None) -> None:
+    """Apply llm_query_finalize hooks from all plugins."""
+    try:
+        plugin_manager.hook.llm_query_finalize(
+            llm_context=llm_context,
+            result=result,
+            error=error,
+        )
+    except Exception as e:
+        logger.error(f"Error in llm_query_finalize hook: {e}")
+        logger.exception("llm_query_finalize hook exception details:")
+
+
 def collect_html_head_extras():
     """Collect and combine HTML head extras from all plugins.
 

changedetectionio/processors/restock_diff/plugins/llm_restock.py

@@ -204,7 +204,7 @@ def get_itemprop_availability_override(content, fetcher_name, fetcher_instance,
 
     try:
         from changedetectionio.llm.evaluator import _runtime_llm_config, accumulate_global_tokens
-        from changedetectionio.llm import client as llm_client
+        from changedetectionio.llm.invocation import llm_completion
     except ImportError as e:
         logger.debug(f"LLM restock fallback: LLM libraries not available ({e})")
         return None
@@ -229,7 +229,10 @@ def get_itemprop_availability_override(content, fetcher_name, fetcher_instance,
         user_prompt += f'\n\nUser notification intent: {llm_intent}'
 
     try:
-        raw, tokens, input_tokens, output_tokens = llm_client.completion(
+        raw, tokens, input_tokens, output_tokens = llm_completion(
+            'restock_extract',
+            watch=None,
+            datastore=datastore,
             model=llm_cfg['model'],
             messages=[
                 {'role': 'system', 'content': SYSTEM_PROMPT},

changedetectionio/templates/_common_fields.html

@@ -112,7 +112,7 @@
                 <td><code>{{ '{{triggered_text}}' }}</code></td>
                 <td>{{ _('Text that tripped the trigger from filters') }}</td>
             </tr>
-            {% if settings_application and settings_application.get('llm', {}).get('model') %}
+            {% if not llm_features_disabled and settings_application and settings_application.get('llm', {}).get('model') %}
             <tr>
                 <td><code>{{ '{{diff}}' }}</code> <small style="opacity:0.6">{{ _('(upgraded)') }}</small></td>
                 <td>{{ _('When AI Change Summary is configured, contains the AI-generated description instead of the raw diff. Falls back to raw diff when not configured.') }}</td>

changedetectionio/templates/base.html

@@ -281,6 +281,7 @@
       </div>
     </dialog>
 
+    {% if not llm_features_disabled %}
     <!-- LLM Not Configured Modal -->
     <dialog id="llm-not-configured-modal" class="modal-dialog" aria-labelledby="llm-not-configured-modal-title">
       <div class="modal-header">
@@ -294,6 +295,7 @@
         <button type="button" class="pure-button" id="close-llm-not-configured-modal">{{ _('Close') }}</button>
       </div>
     </dialog>
+    {% endif %}
 
     <!-- Search Modal -->
     {% if current_user.is_authenticated or not has_password %}

changedetectionio/templates/menu.html

@@ -37,10 +37,12 @@
     </li>
 {% endif %}
     <li class="pure-menu-item menu-collapsible" id="inline-menu-extras-group">
+        {% if not llm_features_disabled %}
         <button class="toggle-button toggle-ai-mode" type="button" title="{{ _('Toggle AI Mode') }}" data-llm-configured="{{ 'true' if llm_configured else 'false' }}" data-llm-settings-url="{{ url_for('settings.settings_page') }}#ai">
             <span class="visually-hidden">{{ _('Toggle AI mode') }}</span>
             {% include "svgs/ai-mode-icon.svg" %}<span class="ai-mode-label">LLM</span>
         </button>
+        {% endif %}
         <button class="toggle-button toggle-light-mode " type="button" title="{{ _('Toggle Light/Dark Mode') }}">
             <span class="visually-hidden">{{ _('Toggle light/dark mode') }}</span>
             <span class="icon-light">

changedetectionio/tests/llm/test_llm_features_disabled.py

@@ -0,0 +1,62 @@
+"""
+Smoke test for the LLM_FEATURES_DISABLED env var.
+
+The env var is intended to hide every LLM/AI surface (settings tab, edit tab,
+base-template AI toggle/modal) for hosted deployments. This test renders the
+three primary pages with the env var set and verifies that none of the
+LLM-related markers leak through.
+"""
+from flask import url_for
+
+
+def _llm_markers_absent(body: bytes, where: str = ''):
+    """All of these strings appear in LLM UI surfaces — none should render."""
+    for marker in (b'AI / LLM', b'toggle-ai-mode', b'llm-not-configured-modal',
+                   b'id="ai-llm"', b'#ai-llm', b'href="#ai"'):
+        if marker in body:
+            idx = body.find(marker)
+            context = body[max(0, idx - 80):idx + len(marker) + 80].decode('utf-8', 'replace')
+            raise AssertionError(f"[{where}] {marker!r} found in body, context: ...{context}...")
+
+
+def test_llm_features_disabled_hides_ui(client, live_server, monkeypatch):
+    monkeypatch.setenv('LLM_FEATURES_DISABLED', 'true')
+
+    # Sanity: helper reports the env var is in effect
+    from changedetectionio.llm.evaluator import is_llm_features_disabled, get_llm_config
+    assert is_llm_features_disabled() is True
+    # get_llm_config() must return None so every `if llm_configured` template hides
+    datastore = client.application.config.get('DATASTORE')
+    assert get_llm_config(datastore) is None
+
+    # 1. Watch list (base.html + menu.html surface)
+    res = client.get(url_for('watchlist.index'))
+    assert res.status_code == 200
+    _llm_markers_absent(res.data, where='watchlist')
+
+    # 2. Settings page (should not have an AI / LLM tab or the LLM tab body)
+    res = client.get(url_for('settings.settings_page'))
+    assert res.status_code == 200
+    _llm_markers_absent(res.data, where='settings')
+
+    # 3. Edit page for a watch (should not have an AI / LLM tab or include_llm_intent body)
+    uuid = datastore.add_watch(url='http://example.com', extras={'title': 'Disabled LLM watch'})
+    res = client.get(url_for('ui.ui_edit.edit_page', uuid=uuid))
+    assert res.status_code == 200
+    _llm_markers_absent(res.data, where='edit')
+    # The watch-edit-only intent textarea should also be absent
+    assert b'name="llm_intent"' not in res.data
+    assert b'name="llm_change_summary"' not in res.data
+
+
+def test_llm_features_enabled_by_default(client, live_server, monkeypatch):
+    """When LLM_FEATURES_DISABLED is unset, the AI / LLM surfaces are still rendered."""
+    monkeypatch.delenv('LLM_FEATURES_DISABLED', raising=False)
+
+    from changedetectionio.llm.evaluator import is_llm_features_disabled
+    assert is_llm_features_disabled() is False
+
+    res = client.get(url_for('settings.settings_page'))
+    assert res.status_code == 200
+    # The AI / LLM settings tab anchor should be present when not disabled
+    assert b'href="#ai"' in res.data

changedetectionio/tests/plugins/test_llm_query_hooks.py

@@ -0,0 +1,136 @@
+"""Tests for llm_query_alter and llm_query_finalize pluggy hooks."""
+import pytest
+
+from changedetectionio.pluggy_interface import hookimpl, plugin_manager
+
+
+class _AlterPlugin:
+    @hookimpl
+    def llm_query_alter(self, llm_context):
+        messages = list(llm_context.get('messages') or [])
+        if messages:
+            messages[-1] = dict(messages[-1])
+            messages[-1]['content'] = (messages[-1].get('content') or '') + ' [altered]'
+        return {'messages': messages, 'max_tokens': 99}
+
+
+class _FinalizePlugin:
+    def __init__(self):
+        self.calls = []
+
+    @hookimpl
+    def llm_query_finalize(self, llm_context, result, error):
+        self.calls.append({
+            'purpose': llm_context.get('purpose'),
+            'app_guid': llm_context.get('app_guid'),
+            'watch_uuid': llm_context.get('watch_uuid'),
+            'result': result,
+            'error': error,
+        })
+
+
+@pytest.fixture
+def alter_plugin():
+    plugin_manager.register(_AlterPlugin(), name='test_llm_alter')
+    yield
+    plugin_manager.unregister(name='test_llm_alter')
+
+
+@pytest.fixture
+def finalize_plugin():
+    plugin = _FinalizePlugin()
+    plugin_manager.register(plugin, name='test_llm_finalize')
+    yield plugin
+    plugin_manager.unregister(name='test_llm_finalize')
+
+
+def test_llm_query_alter_modifies_messages(client, live_server, measure_memory_usage, datastore_path, alter_plugin, monkeypatch):
+    from changedetectionio.llm import invocation as inv
+
+    captured = {}
+
+    def fake_completion(**kwargs):
+        captured.update(kwargs)
+        return 'ok', 10, 6, 4, {'finish_reason': 'stop'}
+
+    monkeypatch.setattr(inv.llm_client, 'completion', fake_completion)
+
+    ds = client.application.config.get('DATASTORE')
+    uuid = ds.add_watch(url='http://example.com', extras={'title': 'Hook test'})
+    watch = ds.data['watching'][uuid]
+
+    text, total, inp, out = inv.llm_completion(
+        'test_purpose',
+        watch=watch,
+        datastore=ds,
+        model='gpt-4o-mini',
+        messages=[{'role': 'user', 'content': 'hello'}],
+    )
+
+    assert text == 'ok'
+    assert total == 10
+    assert '[altered]' in captured['messages'][-1]['content']
+    assert captured['max_tokens'] == 99
+
+
+def test_llm_query_finalize_receives_context_and_result(
+        client, live_server, measure_memory_usage, datastore_path, finalize_plugin, monkeypatch):
+    from changedetectionio.llm import invocation as inv
+
+    def fake_completion(**kwargs):
+        return 'done', 42, 30, 12, {
+            'finish_reason': 'stop',
+            'litellm_response_cost_usd': 0.00123,
+        }
+
+    monkeypatch.setattr(inv.llm_client, 'completion', fake_completion)
+
+    ds = client.application.config.get('DATASTORE')
+    uuid = ds.add_watch(url='http://example.com', extras={'title': 'Finalize test'})
+    watch = ds.data['watching'][uuid]
+    app_guid = ds.data.get('app_guid')
+
+    inv.llm_completion(
+        'evaluate_change',
+        watch=watch,
+        datastore=ds,
+        model='gpt-4o-mini',
+        messages=[{'role': 'user', 'content': 'ping'}],
+    )
+
+    assert len(finalize_plugin.calls) == 1
+    call = finalize_plugin.calls[0]
+    assert call['purpose'] == 'evaluate_change'
+    assert call['app_guid'] == app_guid
+    assert call['watch_uuid'] == uuid
+    assert call['error'] is None
+    assert call['result']['total_tokens'] == 42
+    assert call['result']['input_tokens'] == 30
+    assert call['result']['output_tokens'] == 12
+    assert call['result']['cost_usd'] > 0
+    assert call['result']['litellm_response_cost_usd'] == 0.00123
+
+
+def test_llm_query_finalize_on_error(
+        client, live_server, measure_memory_usage, datastore_path, finalize_plugin, monkeypatch):
+    from changedetectionio.llm import invocation as inv
+
+    def fake_completion(**kwargs):
+        raise RuntimeError('provider down')
+
+    monkeypatch.setattr(inv.llm_client, 'completion', fake_completion)
+
+    ds = client.application.config.get('DATASTORE')
+
+    with pytest.raises(RuntimeError, match='provider down'):
+        inv.llm_completion(
+            'connection_test',
+            watch=None,
+            datastore=ds,
+            model='gpt-4o-mini',
+            messages=[{'role': 'user', 'content': 'x'}],
+        )
+
+    assert len(finalize_plugin.calls) == 1
+    assert finalize_plugin.calls[0]['result'] is None
+    assert str(finalize_plugin.calls[0]['error']) == 'provider down'

changedetectionio/worker.py

@@ -436,7 +436,8 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                         # Also gated on llm_enabled — a disabled LLM can't be spending tokens,
                         # so the budget enforcement shouldn't suppress changes when the user
                         # has explicitly switched LLM off.
-                        _llm_master_enabled = bool(datastore.data['settings']['application'].get('llm_enabled', True))
+                        from changedetectionio.llm.evaluator import is_llm_features_disabled as _is_llm_features_disabled
+                        _llm_master_enabled = bool(datastore.data['settings']['application'].get('llm_enabled', True)) and not _is_llm_features_disabled()
                         _llm_budget_action = datastore.data['settings']['application'].get('llm_budget_action', 'skip_llm')
                         if _llm_master_enabled and _llm_budget_action == 'skip_check':
                             from changedetectionio.llm.evaluator import is_global_token_budget_exceeded