This commit is contained in:
dgtlmoon
2026-07-17 15:29:24 +02:00
parent 3162340ce9
commit c596cf6c14
16 changed files with 449 additions and 203 deletions
@@ -266,9 +266,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
# Resolve the fetcher backend for this watch so we can ask it to launch its own browser
# if it supports that (e.g. CloakBrowser, which runs locally rather than via CDP)
watch = datastore.data['watching'][watch_uuid]
fetcher_name = watch.get_fetch_backend or 'system'
if fetcher_name == 'system':
fetcher_name = datastore.data['settings']['application'].get('fetch_backend', 'html_requests')
# get_fetch_backend is fully resolved (group override / watch / 'system' -> global default).
fetcher_name = watch.get_fetch_backend
browser, playwright_context = await acquire_browser_for_fetcher(fetcher_name, proxy=proxy, keepalive_ms=keepalive_ms)
@@ -66,17 +66,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
extra_notification_tokens=datastore.get_unique_notification_tokens_available()
)
# The global default fetch method is now the "Default browser": the always-present
# built-in engines + the user's saved browsers (no 'system' - this IS the system
# default). Everything else in the app resolves 'system' back to this value.
from changedetectionio.model.browser_config import list_builtin_browsers
default_browser_choices = [(b['id'], b['label']) for b in list_builtin_browsers()]
default_browser_choices += [(cid, e.get('label') or cid)
for cid, e in datastore.browser_config_store.all().items()]
form.application.form.fetch_backend.choices = default_browser_choices
form.application.form.fetch_backend.label.text = gettext('Default browser')
# Accept legacy raw-engine values / ids not in the (dynamic) choice list.
form.application.form.fetch_backend.validate_choice = False
# The global "Default browser" was migrated to the /browsers tab (its per-row radio
# writes settings.application.fetch_backend). Drop the inherited fetch_backend field so a
# settings save never renders, validates, or clobbers the default - /browsers owns it now.
del form.application.form.fetch_backend
# Remove the last option 'System default'
form.application.form.notification_format.choices.pop()
@@ -260,8 +253,18 @@ def construct_blueprint(datastore: ChangeDetectionStore):
# Cost display: only when user configured their own key (not hosted/operator-managed)
llm_show_costs = not llm_env_configured
# Read-only label for the global "Default browser" (managed on the /browsers tab).
from changedetectionio.model.browser_config import list_builtin_browsers
_default_id = datastore.get_default_backend()
_default_entry = datastore.browser_config_store.get(_default_id)
if _default_entry:
default_browser_label = _default_entry.get('label') or _default_id
else:
default_browser_label = dict((b['id'], b['label']) for b in list_builtin_browsers()).get(_default_id, _default_id)
output = render_template("settings.html",
active_plugins=active_plugins,
default_browser_label=default_browser_label,
api_key=datastore.data['settings']['application'].get('api_access_token'),
llm_config=llm_config,
llm_env_configured=llm_env_configured,
@@ -100,14 +100,18 @@
</div>
<div class="tab-pane-inner" id="fetching">
<div class="pure-control-group inline-radio">
{{ render_field(form.application.form.fetch_backend, class="fetch-backend") }}
<div class="pure-control-group">
<label>{{ _('Default browser') }}</label>
<div class="default-browser-summary" style="margin:0.3em 0;">
<strong style="margin-right:0.75em;">{{ default_browser_label }}</strong>
<a class="pure-button button-small" href="{{ url_for('ui.browser_config.browsers_overview') }}">{{ _('Manage browsers &amp; set default')|safe }}</a>
</div>
<span class="pure-form-message-inline">
<p>{{ _('Use the <strong>Basic</strong> method (default) where your watched sites don\'t need Javascript to render.')|safe }}</p>
<p>{{ _('The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var \'WEBDRIVER_URL\'.')|safe }}</p>
<p>{{ _('Every watch uses this browser unless it (or its group) picks another one.')|safe }}</p>
<p>{{ _('Add screen sizes, languages, timezones and choose the default on the <a href="%(url)s">Browsers</a> page.', url=url_for('ui.browser_config.browsers_overview'))|safe }}</p>
</span>
</div>
<fieldset class="pure-group" id="webdriver-override-options" data-visible-for="application-fetch_backend=html_webdriver">
<fieldset class="pure-group" id="webdriver-override-options">
<div class="pure-form-message-inline">
<strong>{{ _('If you\'re having trouble waiting for the page to be fully rendered (text missing etc), try increasing the \'wait\' time here.') }}</strong>
<br>
@@ -208,12 +208,15 @@ def construct_blueprint(datastore: ChangeDetectionStore):
@browser_config_blueprint.route("/browsers/set-default/<string:config_id>", methods=['POST'])
@login_optionally_required
def browser_config_set_default(config_id):
# "Default" is the global system fetch_backend - the single source of truth that a
# watch/group set to 'system' resolves to. This is settable from here or from Settings.
# "Default browser" is the global settings.application.fetch_backend - the single source
# of truth a watch/group set to 'system' resolves to. This /browsers tab is now the only
# place it's set (the Settings page shows it read-only). Only usable (ready-to-use)
# built-in engines or saved browser configs may be the default.
from changedetectionio.model.browser_config import list_builtin_browsers
builtins = {b['id'] for b in list_builtin_browsers()}
if config_id in builtins or datastore.browser_config_store.get(config_id):
datastore.data['settings']['application']['fetch_backend'] = config_id
logger.debug(f"Default browser (settings.application.fetch_backend) set to '{config_id}'")
flash(gettext("Default browser set"))
else:
flash(gettext("Browser config not found"), 'error')
@@ -8,16 +8,27 @@
<p class="pure-form-message-inline">
{{ _('Save a browser once (its screen size, language, timezone…) and reuse it on any watch or group. Add a variation from one of the available browsers below.') }}
</p>
<p class="pure-form-message-inline">
{{ _('The <strong>Default browser</strong> (chosen with the radio button below) is used by every watch unless the watch — or its group — picks another one. This is the same value shown on the <a href="%(url)s#fetching">Settings → Fetching</a> page.', url=url_for('settings.settings_page'))|safe }}
</p>
{# ---- User-created browser variations ---- #}
<h5 style="margin-top:1.5em">{{ _('Your browsers') }}</h5>
{% if browser_configs %}
<table class="pure-table" style="width:100%; margin-top:0.5em;">
<thead><tr><th>{{ _('Name') }}</th><th>{{ _('Setup') }}</th><th></th><th></th></tr></thead>
<table class="pure-table browsers-table" style="width:100%; margin-top:0.5em;">
<thead><tr><th style="width:5em; text-align:center">{{ _('Default') }}</th><th>{{ _('Name') }}</th><th>{{ _('Setup') }}</th><th></th></tr></thead>
<tbody>
{% for cid, entry in browser_configs.items() %}
{% set bc = entry.get('browser_config') or {} %}
<tr>
<tr class="{% if cid == default_browser_id %}is-default{% endif %}">
<td style="text-align:center">
<form method="POST" class="set-default-form" action="{{ url_for('ui.browser_config.browser_config_set_default', config_id=cid) }}">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
<input type="radio" class="browser-default-radio" name="browser_default" value="{{ cid }}"
title="{{ _('Set as the default browser') }}"
{% if cid == default_browser_id %}checked{% endif %}>
</form>
</td>
<td>
<strong>{{ entry.get('label') }}</strong>
<span class="browser-chip">{{ entry.get('base_fetcher') }}</span>
@@ -30,14 +41,6 @@
{% if bc.get('timezone_id') %}<span class="browser-chip">{{ bc.timezone_id }}</span>{% endif %}
{% if bc.get('block_resource_types') %}<span class="browser-chip">{{ _('no') }} {{ bc.block_resource_types|join(', ') }}</span>{% endif %}
</td>
<td style="text-align:right; white-space:nowrap">
{% if cid != default_browser_id %}
<form method="POST" action="{{ url_for('ui.browser_config.browser_config_set_default', config_id=cid) }}" style="display:inline">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
<button class="pure-button button-small" type="submit">{{ _('Make default') }}</button>
</form>
{% endif %}
</td>
<td style="text-align:right; white-space:nowrap">
<a class="pure-button button-small" href="{{ url_for('ui.browser_config.browser_config_edit', config_id=cid) }}">{{ _('Edit') }}</a>
<form method="POST" action="{{ url_for('ui.browser_config.browser_config_remove', config_id=cid) }}" style="display:inline"
@@ -57,12 +60,23 @@
{# ---- Available base browsers - each offers "Add variation" ---- #}
<h5 style="margin-top:2em">{{ _('Available browsers') }}</h5>
{% if base_fetchers %}
<table class="pure-table" style="width:100%; margin-top:0.5em;">
<thead><tr><th>{{ _('Browser') }}</th><th>{{ _('Capabilities') }}</th><th></th><th></th></tr></thead>
<table class="pure-table browsers-table" style="width:100%; margin-top:0.5em;">
<thead><tr><th style="width:5em; text-align:center">{{ _('Default') }}</th><th>{{ _('Browser') }}</th><th>{{ _('Capabilities') }}</th><th></th><th></th></tr></thead>
<tbody>
{% for f in base_fetchers %}
{% set fbc = f.browser_config or {} %}
<tr>
<tr class="{% if f.ready_to_use and f.name == default_browser_id %}is-default{% endif %}">
<td style="text-align:center">
{# Only ready-to-use engines can be the default (base-only engines aren't usable directly) #}
{% if f.ready_to_use %}
<form method="POST" class="set-default-form" action="{{ url_for('ui.browser_config.browser_config_set_default', config_id=f.name) }}">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
<input type="radio" class="browser-default-radio" name="browser_default" value="{{ f.name }}"
title="{{ _('Set as the default browser') }}"
{% if f.name == default_browser_id %}checked{% endif %}>
</form>
{% endif %}
</td>
<td>
<strong>{{ f.description }}</strong>
{% if not f.ready_to_use %}<span class="browser-chip">{{ _('needs setup') }}</span>{% endif %}
@@ -74,14 +88,8 @@
{% endfor %}
</td>
<td style="text-align:right; white-space:nowrap">
{# ready-to-use engines are usable directly: can be edited + made default #}
{# ready-to-use engines are usable directly: they can be edited #}
{% if f.ready_to_use %}
{% if f.name != default_browser_id %}
<form method="POST" action="{{ url_for('ui.browser_config.browser_config_set_default', config_id=f.name) }}" style="display:inline">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
<button class="pure-button button-small" type="submit">{{ _('Make default') }}</button>
</form>
{% endif %}
<a class="pure-button button-small" href="{{ url_for('ui.browser_config.browser_config_edit', config_id=f.name) }}">{{ _('Edit') }}</a>
{% endif %}
</td>
@@ -102,6 +110,21 @@
</div>
</div>
<script>
// Radio buttons live in per-row mini-forms (so each posts its own set-default action). They
// share a name so they read as one group; on change we clear the others for instant feedback,
// then submit — the server persists the choice and the reloaded page reflects it authoritatively.
document.querySelectorAll('.browser-default-radio').forEach(function (radio) {
radio.addEventListener('change', function () {
if (!radio.checked) return;
document.querySelectorAll('.browser-default-radio').forEach(function (other) {
if (other !== radio) other.checked = false;
});
radio.closest('form').submit();
});
});
</script>
<style>
.browser-chip {
display: inline-block;
@@ -113,6 +136,9 @@
white-space: nowrap;
}
.browser-chip--default { background: #d4edda; color: #155724; }
.browsers-table tr.is-default { background: rgba(21,87,36,0.06); }
.browser-default-radio { transform: scale(1.3); cursor: pointer; }
.set-default-form { margin: 0; display: inline; }
.browser-empty {
margin-top: 0.75em;
padding: 2em;
+14 -30
View File
@@ -106,18 +106,15 @@ def _log_fetcher_capabilities(fetcher_class, backend_name, uuid=None):
def resolve_content_fetcher(watch, datastore):
"""Single source of truth for resolving which content fetcher a watch should use.
"""Build the concrete content-fetcher class + config for a watch.
Resolution order (room to grow):
1. Watch-level `fetch_backend`
2. (future) group-level default
3. System/global default from application settings
Also collapses the special backend forms into a concrete fetcher:
- 'system' -> global application default
*Which* browser/engine is selected is owned by the Watch model (watch.get_fetch_backend:
PDF / group override / watch / 'system' -> global Default browser). This function takes that
resolved selector and turns it into a fetcher instance, collapsing the special forms:
- a user browser-config id -> its base_fetcher engine + its FetcherConfig
- 'extra_browser_<key>' -> html_webdriver + custom connection URL
- is_pdf watch -> forced html_requests (browser PDF support incomplete)
- html_webdriver + browser_steps -> playwright override (puppeteer steps incomplete)
- a deleted browser-config id -> raises BrowserConfigDoesntExist
Returns:
tuple: (fetcher_class, backend_name, custom_browser_connection_url, browser_config)
@@ -126,22 +123,15 @@ def resolve_content_fetcher(watch, datastore):
resolved FetcherConfig to inject as `.browser_config`.
"""
this_module = sys.modules[__name__]
from changedetectionio.model.browser_config import (
FetcherConfig, resolve_browser_config_override, BrowserConfigDoesntExist,
)
from changedetectionio.model.browser_config import FetcherConfig, BrowserConfigDoesntExist
# Default behaviour = empty config (built-in engines / system default).
browser_config = FetcherConfig()
# Selection order: a group override wins, else the watch's own selector value.
# The value is either a built-in engine name ('html_requests', 'html_webdriver',
# 'extra_browser_*'), the sentinel 'system', or the stable id of a user browser config.
override = resolve_browser_config_override(watch, datastore)
selected = override['config_id'] if override else watch.get('fetch_backend', 'system')
# 'system' -> the global default, which may itself be a browser-config id or an engine name.
if not selected or selected == 'system':
selected = datastore.data['settings']['application'].get('fetch_backend')
# THE single resolved selector for this watch (PDF / group override / watch / 'system' ->
# global default) - the Watch owns this chain so every codepath agrees. The value is a
# built-in engine name, 'extra_browser_*', or the stable id of a user browser config.
selected = watch.get_fetch_backend
store = getattr(datastore, 'browser_config_store', None)
entry = store.get(selected) if (store and selected) else None
@@ -170,15 +160,9 @@ def resolve_content_fetcher(watch, datastore):
prefer_fetch_backend = 'html_webdriver'
custom_browser_connection_url = connection[0].get('browser_connection_url')
# PDF should be html_requests because playwright will serve it up (so far) in an embedded page
# @todo https://github.com/dgtlmoon/changedetection.io/issues/2019
if getattr(watch, 'is_pdf', False):
logger.warning(
f"Watch {watch.get('uuid')} is_pdf detected (content-type/url) - forcing the "
f"'html_requests' fetcher because browser support isn't complete yet for "
f"saving/downloading the PDF. Overriding requested backend '{prefer_fetch_backend}'."
)
prefer_fetch_backend = "html_requests"
# PDF watches are already forced to 'html_requests' by Watch.get_fetch_backend (playwright
# can't render a PDF in-page yet - @todo https://github.com/dgtlmoon/changedetection.io/issues/2019),
# so no extra handling is needed here.
# Grab the right kind of 'fetcher' class (playwright, requests, plugin-provided, etc)
if prefer_fetch_backend and hasattr(this_module, prefer_fetch_backend):
+5 -1
View File
@@ -1091,7 +1091,11 @@ class globalSettingsApplicationForm(commonSettingsForm):
render_kw={"placeholder": os.getenv('BASE_URL', _l('Not set'))}
)
empty_pages_are_a_change = BooleanField(_l('Treat empty pages as a change?'), default=False)
fetch_backend = RadioField(_l('Fetch Method'), default="html_requests", choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
# NOTE: the global "Default browser" is NOT a field here anymore - all browser choice was
# migrated to the /browsers tab (set-default writes settings.application.fetch_backend). The
# inherited commonSettingsForm.fetch_backend field is del()'d from this form in the settings
# blueprint so a settings save never touches the default. Watches still pick a browser via
# their own commonSettingsForm.fetch_backend (processor_text_json_diff_form).
global_ignore_text = StringListField(_l('Ignore Text'), [ValidateListRegex()])
global_subtractive_selectors = StringListField(_l('Remove elements'), [ValidateCSSJSONXPATHInput(allow_json=False)])
ignore_whitespace = BooleanField(_l('Ignore whitespace'))
+103 -45
View File
@@ -361,62 +361,120 @@ class model(EntityPersistenceMixin, watch_base):
def is_source_type_url(self):
return self.get('url', '').startswith('source:')
@property
def browser_config_store(self):
"""A read-only, file-backed view of browsers.json for THIS watch.
The Watch only holds the raw settings data dict (self._datastore), not the full
ChangeDetectionStore, so it can't reach the shared browser_config_store directly. The
per-path registry hands back that same shared (mtime-cached) instance so the watch can
self-resolve its browser config (id -> engine, group overrides) without threading the
store through every call, and without re-reading browsers.json per watch per render.
"""
from changedetectionio.model.browser_config import get_browser_config_store
if not self._datastore_path:
return None
return get_browser_config_store(self._datastore_path)
@property
def _global_default_fetch_backend(self):
"""The global 'Default browser' (settings.application.fetch_backend) - the single source
of truth that BOTH /browsers (per-row radio) and Settings->Fetching write. May be a
built-in engine name or a user browser-config id."""
if self._datastore:
return self._datastore['settings']['application'].get('fetch_backend', 'html_requests')
return 'html_requests'
@property
def browser_config_override(self):
"""If a group/tag overrides this watch's browser config, describe it, else None.
Uses the group's dedicated enabler (browser_config_overrides_watch), NOT the coarse
legacy `overrides_watch` flag. Tags are checked in the watch's tag-list order; the first
one whose enabler is on and whose selected config still exists (a user browser OR a
built-in engine) wins. Returns {group_uuid, group_title, config_id, label} or None.
"""
from changedetectionio.model.browser_config import list_builtin_browsers
if not self._datastore:
return None
tags = self._datastore['settings']['application'].get('tags', {})
store = self.browser_config_store
builtins = {b['id']: b for b in list_builtin_browsers()}
for tag_uuid in (self.get('tags') or []):
tag = tags.get(tag_uuid)
if not tag:
continue
if tag.get('browser_config_overrides_watch') and tag.get('browser_config'):
cid = tag.get('browser_config')
entry = store.get(cid) if store else None
if entry:
label = entry.get('label')
elif cid in builtins:
label = builtins[cid]['label']
else:
continue # dangling / unknown - don't apply, fall through
return {'group_uuid': tag_uuid, 'group_title': tag.get('title'),
'config_id': cid, 'label': label}
return None
@property
def get_fetch_backend(self):
"""THE fully-resolved browser/engine selector this watch fetches with.
Single source of truth for the whole app (content fetcher, watchlist status icon,
capability checks, browser steps). Resolution order:
1. PDF watches -> 'html_requests' (chrome/playwright can't render a PDF in-page;
we fetch it and pdf2html the text instead)
2. group override -> the overriding group's browser config id
3. the watch's own `fetch_backend`
4. 'system' / unset -> the global Default browser (_global_default_fetch_backend)
Never returns 'system'. The value is a user browser-config id, a built-in engine name
('html_requests'/'html_webdriver'/...), or 'extra_browser_<key>'. Map it to the concrete
engine with .resolved_fetch_engine; build the fetcher class/config via
content_fetchers.resolve_content_fetcher (which consumes this).
"""
Get the fetch backend for this watch with special case handling.
CHAIN RESOLUTION OPPORTUNITY:
Currently returns watch.fetch_backend directly, but doesn't implement
Watch Tag Global resolution chain. With Pydantic:
@computed_field
def resolved_fetch_backend(self) -> str:
# Special case: PDFs always use html_requests
if self.is_pdf:
return 'html_requests'
# Watch override
if self.fetch_backend and self.fetch_backend != 'system':
return self.fetch_backend
# Tag override (first tag with overrides_watch=True wins)
for tag_uuid in self.tags:
tag = self._datastore.get_tag(tag_uuid)
if tag.overrides_watch and tag.fetch_backend:
return tag.fetch_backend
# Global default
return self._datastore.settings.fetch_backend
"""
# Maybe also if is_image etc?
# This is because chrome/playwright wont render the PDF in the browser and we will just fetch it and use pdf2html to see the text.
uuid = self.get('uuid')
if self.is_pdf:
logger.debug(f"fetch_backend: watch {uuid} is_pdf -> forced 'html_requests'")
return 'html_requests'
return self.get('fetch_backend')
override = self.browser_config_override
if override:
logger.debug(f"fetch_backend: watch {uuid} uses group '{override.get('group_title')}' "
f"override -> '{override['config_id']}'")
return override['config_id']
selected = self.get('fetch_backend') or 'system'
if selected == 'system':
default = self._global_default_fetch_backend
logger.debug(f"fetch_backend: watch {uuid} 'system' -> global default '{default}'")
return default
logger.debug(f"fetch_backend: watch {uuid} -> watch-level '{selected}'")
return selected
@property
def resolved_fetch_engine(self):
"""The concrete engine name behind .get_fetch_backend - maps a browser-config id to its
base_fetcher, or passes a built-in engine name straight through. Used for capability
checks and the watchlist status icon so they agree with what actually fetches the page."""
value = self.get_fetch_backend
store = self.browser_config_store
entry = store.get(value) if (store and value) else None
if entry:
return entry.get('base_fetcher') or 'html_webdriver'
return value or 'html_requests'
@property
def fetcher_supports_screenshots(self):
"""Return True if the fetcher configured for this watch supports screenshots.
Resolves 'system' via self._datastore, then checks supports_screenshots on
the actual fetcher class. Works for built-in and plugin fetchers alike.
"""
"""True if the fetcher this watch resolves to supports screenshots. Honours group
overrides + browser-config ids (via .resolved_fetch_engine). Works for built-in and
plugin fetchers alike."""
from changedetectionio import content_fetchers
from changedetectionio.content_fetchers.base import FetcherCapabilities
from changedetectionio.model.browser_config import base_fetcher_for
# get_fetch_backend handles is_pdf; base_fetcher_for maps a browser-config id / 'system'
# to the concrete engine name (and PDF -> html_requests is preserved by get_fetch_backend).
fetch_backend = self.get_fetch_backend
if fetch_backend == 'html_requests':
fetcher_name = 'html_requests'
else:
fetcher_name = base_fetcher_for(fetch_backend, self._datastore)
# Shared capability model - handles an unknown/None fetcher class (all-False)
fetcher_class = getattr(content_fetchers, fetcher_name, None)
fetcher_class = getattr(content_fetchers, self.resolved_fetch_engine, None)
return FetcherCapabilities.from_fetcher(fetcher_class).supports_screenshots
@property
+60 -79
View File
@@ -121,8 +121,9 @@ class BrowserConfigEntry(BaseModel):
"""One browsers.json entry (the id is the dict key, not stored in the entry).
Note: there is deliberately no `is_default` here. The default browser is the global
application `fetch_backend` (a single source of truth that can point at a built-in engine
OR a browser-config id), so it doesn't live per-entry - see base_fetcher_for().
settings.application.fetch_backend (a single source of truth that can point at a built-in
engine OR a browser-config id), so it doesn't live per-entry - see
ChangeDetectionStore.get_default_backend() / Watch.get_fetch_backend.
"""
label: str = ''
base_fetcher: str = 'html_webdriver'
@@ -135,27 +136,44 @@ class BrowserConfigStore:
def __init__(self, datastore_path, lock):
self._path = os.path.join(datastore_path, 'browsers.json')
self._lock = lock
# mtime-keyed cache of the parsed file. all()/get() are hit per-watch on every watchlist
# render + fetch resolution, so re-reading + re-parsing browsers.json each time is real
# amplification (cf. the favicon-glob fix). Cache the parsed dict and only re-read when
# the file's mtime changes - picks up edits (save bumps mtime) without staleness.
self._cache = None
self._cache_mtime = None
# ---- low level load/save ----
def all(self):
"""Raw dict {id: entry-dict}. Empty dict when the file is absent."""
if not path.isfile(self._path):
"""Raw dict {id: entry-dict}. Empty dict when the file is absent. mtime-cached."""
try:
mtime = os.path.getmtime(self._path)
except OSError:
# File absent (or unreadable) - nothing configured yet.
self._cache, self._cache_mtime = {}, None
return {}
if self._cache is not None and self._cache_mtime == mtime:
return self._cache
try:
if HAS_ORJSON:
with open(self._path, 'rb') as f:
return orjson.loads(f.read()) or {}
with open(self._path, encoding='utf-8') as f:
return json.load(f) or {}
data = orjson.loads(f.read()) or {}
else:
with open(self._path, encoding='utf-8') as f:
data = json.load(f) or {}
except Exception as e:
logger.error(f"Could not load browsers.json: {e}")
return {}
self._cache, self._cache_mtime = data, mtime
return data
def _save(self, configs):
# Deferred import avoids a model -> store import cycle at module load.
from changedetectionio.store.file_saving_datastore import save_json_atomic
with self._lock:
save_json_atomic(self._path, configs, label="browsers")
# Invalidate so the next all()/get() re-reads (its mtime will differ anyway).
self._cache, self._cache_mtime = None, None
# ---- CRUD ----
def get(self, config_id):
@@ -217,28 +235,24 @@ class BrowserConfigStore:
return FetcherConfig(**(raw.get('browser_config') or {}))
def base_fetcher_for(value, datastore):
"""Map any fetch_backend value to the concrete engine/fetcher name.
# One BrowserConfigStore instance per datastore path, so the mtime cache is shared by everything
# reading browsers.json (the ChangeDetectionStore and every Watch, which only holds the data dict
# + its path). The ChangeDetectionStore registers its own (lock-bearing) instance here so writes
# and the watch-side reads go through the same cache.
_STORE_REGISTRY = {}
Accepts a user browser-config id, a built-in engine name ('html_requests'/'html_webdriver'/
'extra_browser_*'), or the sentinel 'system'. This is the single place the rest of the
codebase should go through when it needs the *engine* behind a watch/global fetch_backend
(capability checks, screenshot support, etc.). 'system' resolves to the global default,
which may itself be a browser-config id.
"""
# `datastore` may be the ChangeDetectionStore (has .browser_config_store and .data) or the
# raw settings data dict (Watch._datastore). Tolerate both; browser-config lookups are only
# possible when the store object is available.
store = getattr(datastore, 'browser_config_store', None)
data = getattr(datastore, 'data', datastore)
entry = store.get(value) if (store and value) else None
if entry:
return entry.get('base_fetcher') or 'html_webdriver'
if not value or value == 'system':
gd = data['settings']['application'].get('fetch_backend', 'html_requests')
gd_entry = store.get(gd) if (store and gd) else None
return (gd_entry.get('base_fetcher') if gd_entry else gd) or 'html_requests'
return value
def register_browser_config_store(datastore_path, store):
_STORE_REGISTRY[datastore_path] = store
def get_browser_config_store(datastore_path):
"""The shared BrowserConfigStore for a datastore path (created read-only if none registered)."""
store = _STORE_REGISTRY.get(datastore_path)
if store is None:
store = BrowserConfigStore(datastore_path, lock=None)
_STORE_REGISTRY[datastore_path] = store
return store
def list_builtin_browsers():
@@ -278,20 +292,24 @@ def _system_default_label(datastore):
return gettext('Default (system settings)')
def resolve_watch_fetcher_engine(watch, datastore):
"""The concrete engine name that will actually fetch this watch.
# --- Thin free-function delegators --------------------------------------------------------
# The resolution chain (PDF / group override / watch / 'system' -> global default) lives on the
# Watch model now (Watch.get_fetch_backend & friends) - only watches fetch, so the watch owns
# "what do I fetch with?". These wrappers keep the historical (watch, datastore) call shape for
# templates/blueprints/tests; `datastore` is accepted but unused (the Watch self-resolves).
Single place that mirrors resolve_content_fetcher's *selection*: a group override wins,
else the watch's own browser selection (a browser-config id or engine name or 'system'),
then mapped to the underlying engine. Used by capability checks and the watchlist status
icon so they all agree with what fetches the page.
"""
override = resolve_browser_config_override(watch, datastore)
selected = override['config_id'] if override else watch.get('fetch_backend', 'system')
return base_fetcher_for(selected, datastore)
def resolve_watch_fetcher_engine(watch, datastore=None):
"""The concrete engine name that will actually fetch this watch. See Watch.resolved_fetch_engine."""
return watch.resolved_fetch_engine
def resolve_watch_browser_display(watch, datastore):
def resolve_browser_config_override(watch, datastore=None):
"""If a group/tag overrides this watch's browser config, describe it, else None.
See Watch.browser_config_override."""
return watch.browser_config_override
def resolve_watch_browser_display(watch, datastore=None):
"""Display info for the watchlist status icon: which browser a watch effectively uses.
Returns dict: {engine, browser_type, label, is_named, group_title} where `label` is the
@@ -300,13 +318,10 @@ def resolve_watch_browser_display(watch, datastore):
override supplies it.
"""
from changedetectionio import content_fetchers
store = getattr(datastore, 'browser_config_store', None)
override = resolve_browser_config_override(watch, datastore)
selected = override['config_id'] if override else watch.get('fetch_backend', 'system')
if not selected or selected == 'system':
selected = datastore.data['settings']['application'].get('fetch_backend', 'html_requests')
store = watch.browser_config_store
override = watch.browser_config_override
selected = watch.get_fetch_backend
entry = store.get(selected) if (store and selected) else None
if entry:
engine = entry.get('base_fetcher') or 'html_webdriver'
@@ -326,37 +341,3 @@ def resolve_watch_browser_display(watch, datastore):
'is_named': is_named,
'group_title': override['group_title'] if override else None,
}
def resolve_browser_config_override(watch, datastore):
"""If a group/tag overrides this watch's browser config, describe it, else None.
Uses the group's own dedicated enabler (browser_config_overrides_watch), NOT the coarse
legacy `overrides_watch` flag. Tags are checked in the watch's tag-list order; the first
one whose enabler is on and whose selected config still exists wins (deterministic).
Returns: {group_uuid, group_title, config_id, label} or None.
"""
tags = datastore.data['settings']['application'].get('tags', {})
builtins = {b['id']: b for b in list_builtin_browsers()}
for tag_uuid in (watch.get('tags') or []):
tag = tags.get(tag_uuid)
if not tag:
continue
if tag.get('browser_config_overrides_watch') and tag.get('browser_config'):
cid = tag.get('browser_config')
entry = datastore.browser_config_store.get(cid)
# The override may point at a user browser (uuid) OR a built-in engine (id == name).
if entry:
label = entry.get('label')
elif cid in builtins:
label = builtins[cid]['label']
else:
continue # dangling / unknown - don't apply
return {
'group_uuid': tag_uuid,
'group_title': tag.get('title'),
'config_id': cid,
'label': label,
}
return None
+11 -1
View File
@@ -61,12 +61,22 @@ class ChangeDetectionStore(DatastoreUpdatesMixin, FileSavingDataStore):
# Named browser configs (browsers.json) - optional, backup-friendly, lazy-loaded.
# Set here in __init__ so it exists on every path (fresh install included).
from changedetectionio.model.browser_config import BrowserConfigStore
from changedetectionio.model.browser_config import BrowserConfigStore, register_browser_config_store
self.browser_config_store = BrowserConfigStore(self.datastore_path, self.lock)
# Share this (lock-bearing, mtime-cached) instance with Watch objects, which only hold
# the data dict + their path and reach the store via the per-path registry.
register_browser_config_store(self.datastore_path, self.browser_config_store)
self.save_version_copy_json_db(version_tag)
self.reload_state(datastore_path=datastore_path, include_default_watches=include_default_watches, version_tag=version_tag)
def get_default_backend(self):
"""The global "Default browser" (settings.application.fetch_backend) - the single source
of truth a watch/group set to 'system' resolves to. Set only on the /browsers tab
(its per-row radio); the Settings page shows it read-only. May be a built-in engine name
or a user browser-config id. See Watch.get_fetch_backend for the per-watch resolution."""
return self.__data['settings']['application'].get('fetch_backend', 'html_requests')
def save_version_copy_json_db(self, version_tag):
"""
Create version-tagged backup of changedetection.json.
+40
View File
@@ -871,4 +871,44 @@ class DatastoreUpdatesMixin:
restock.pop('prev_price', None)
watch.commit()
def update_34(self):
"""Make the global 'Default browser' a concrete, valid selection for the /browsers tab.
All browser choice is managed on /browsers now; the default is
settings.application.fetch_backend (the single source of truth that the per-row radio
writes and every watch/group set to 'system' resolves to). Older installs may hold a
blank/missing value, the sentinel 'system', or a browser-config id that has since been
deleted - any of which would leave the /browsers "Default" radio with nothing selected
(and a watch on 'system' with no concrete engine). Normalise those to a concrete built-in
engine, honouring DEFAULT_FETCH_BACKEND (the same env var fresh installs use), else
'html_requests'.
Concrete values already stored - a built-in engine name (e.g. 'html_webdriver'), an
'extra_browser_*' key, or a still-existing saved browser-config id - are left untouched.
Idempotent.
"""
app = self.data['settings']['application']
current = app.get('fetch_backend')
def _is_valid_default(value):
if not value or value == 'system':
return False
if value.startswith('extra_browser_'):
return True
# A saved browser config?
if self.browser_config_store.get(value):
return True
# A built-in engine that actually exists in this build?
from changedetectionio import content_fetchers
return hasattr(content_fetchers, value)
if _is_valid_default(current):
return # already a concrete, resolvable default - nothing to do
default = os.getenv('DEFAULT_FETCH_BACKEND', 'html_requests') or 'html_requests'
app['fetch_backend'] = default
logger.info(
f"update_34: normalised global Default browser (fetch_backend) from '{current}' to '{default}'"
)
@@ -19,7 +19,6 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
url_for("settings.settings_page"),
data={"application-empty_pages_are_a_change": "",
"requests-time_between_check-minutes": 180,
'application-fetch_backend': "html_webdriver",
'requests-extra_browsers-0-browser_connection_url': 'ws://sockpuppetbrowser-custom-url:3000',
'requests-extra_browsers-0-browser_name': custom_browser_name
},
@@ -28,6 +27,14 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
assert b"Settings updated." in res.data
# The global "Default browser" is now chosen on the /browsers tab (was the settings-page
# application-fetch_backend radio). Make html_webdriver the default.
res = client.post(
url_for("ui.browser_config.browser_config_set_default", config_id="html_webdriver"),
follow_redirects=True
)
assert b"Default browser set" in res.data
# Add our URL to the import page
uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -17,7 +17,6 @@ def test_fetch_webdriver_content(client, live_server, measure_memory_usage, data
data={
"application-empty_pages_are_a_change": "",
"requests-time_between_check-minutes": 180,
'application-fetch_backend': "html_webdriver",
'application-ui-favicons_enabled': "y",
},
follow_redirects=True
@@ -25,6 +24,14 @@ def test_fetch_webdriver_content(client, live_server, measure_memory_usage, data
assert b"Settings updated." in res.data
# The global "Default browser" is now set on the /browsers tab (was application-fetch_backend
# on the settings page). Make html_webdriver the default so 'system' watches use it.
res = client.post(
url_for("ui.browser_config.browser_config_set_default", config_id="html_webdriver"),
follow_redirects=True
)
assert b"Default browser set" in res.data
# Add our URL to the import page
res = client.post(
url_for("imports.import_page"),
@@ -195,11 +195,18 @@ def test_restock_detection(client, live_server, measure_memory_usage, datastore_
url_for("settings.settings_page"),
data={"application-empty_pages_are_a_change": "y",
"requests-time_between_check-minutes": 180,
'application-fetch_backend': fetch_backend,
},
follow_redirects=True
)
# The global "Default browser" is now set on the /browsers tab (was the settings-page
# application-fetch_backend radio).
res = client.post(
url_for("ui.browser_config.browser_config_set_default", config_id=fetch_backend),
follow_redirects=True
)
assert b"Default browser set" in res.data
#####################
# Set this up for when we remove the notification from the watch, it should fallback with these details
res = client.post(
@@ -152,7 +152,9 @@ def test_check_access_control(app, client, live_server, measure_memory_usage, da
assert b"Import" in res.data
assert b"/logout" in res.data
assert b"time_between_check-minutes" in res.data
assert b"fetch_backend" in res.data
# The global browser default moved to the /browsers tab; the settings page now shows it
# read-only as "Default browser" (the editable radio was removed).
assert b"Default browser" in res.data
##################################################
# Remove password button, and check that it worked
@@ -161,7 +163,6 @@ def test_check_access_control(app, client, live_server, measure_memory_usage, da
url_for("settings.settings_page"),
data={
"requests-time_between_check-minutes": 180,
"application-fetch_backend": "html_webdriver",
"application-removepassword_button": "Remove password"
},
follow_redirects=True,
@@ -352,6 +352,118 @@ def test_group_browser_config_override(client, live_server, measure_memory_usage
assert b"French mobile" in res.data # from group ...
def test_settings_default_browser_is_readonly_and_browsers_tab_owns_it(client, live_server, measure_memory_usage, datastore_path):
"""All browser choice moved to /browsers. The Settings->Fetching page shows the global
Default browser read-only (no editable radio) and a settings save must NOT clobber it."""
datastore = client.application.config.get('DATASTORE')
_add_browser(client, label="Settings Default", viewport_width=800)
cid = list(datastore.browser_config_store.all())[0]
client.post(url_for("ui.browser_config.browser_config_set_default", config_id=cid), follow_redirects=True)
assert datastore.data['settings']['application']['fetch_backend'] == cid
# Settings page shows the read-only summary (label + link to /browsers) and NO editable radio.
res = client.get(url_for("settings.settings_page"))
assert b"Default browser" in res.data
assert b"Settings Default" in res.data
assert url_for("ui.browser_config.browsers_overview").encode() in res.data
assert b'name="application-fetch_backend"' not in res.data
# A settings save (the form no longer carries fetch_backend) leaves the default untouched.
res = client.post(url_for("settings.settings_page"),
data={"requests-time_between_check-minutes": 180,
"application-empty_pages_are_a_change": ""},
follow_redirects=True)
assert b"Settings updated." in res.data
assert datastore.data['settings']['application']['fetch_backend'] == cid
def test_browsers_overview_default_radio(client, live_server, measure_memory_usage, datastore_path):
"""Each usable browser row exposes a radio to set it as default; the current default is checked."""
import re
datastore = client.application.config.get('DATASTORE')
_add_browser(client, label="Radio Mobile", viewport_width=390)
cid = list(datastore.browser_config_store.all())[0]
client.post(url_for("ui.browser_config.browser_config_set_default", config_id=cid), follow_redirects=True)
res = client.get(url_for("ui.browser_config.browsers_overview"))
assert b'class="browser-default-radio"' in res.data
# Built-in engines each have a radio too (usable ones)
assert b'value="html_webdriver"' in res.data
# The checked radio is our default config id
checked = re.findall(rb'<input type="radio"[^>]*?>', res.data)
assert any(cid.encode() in tag and b'checked' in tag for tag in checked)
def test_watch_get_fetch_backend_resolution_chain(client, live_server, measure_memory_usage, datastore_path):
"""Watch.get_fetch_backend owns the whole chain: PDF / watch / 'system'->global / group override."""
datastore = client.application.config.get('DATASTORE')
# Global Default browser = html_webdriver (what /browsers set-default writes)
datastore.data['settings']['application']['fetch_backend'] = 'html_webdriver'
uuid = datastore.add_watch(url="https://example.com")
watch = datastore.data['watching'][uuid]
# 'system' resolves to the global default (and maps to the same engine)
watch['fetch_backend'] = 'system'
assert watch.get_fetch_backend == 'html_webdriver'
assert watch.resolved_fetch_engine == 'html_webdriver'
# A watch-level choice wins over the global default
watch['fetch_backend'] = 'html_requests'
assert watch.get_fetch_backend == 'html_requests'
# PDF forces html_requests regardless of the selected browser
watch['fetch_backend'] = 'html_webdriver'
watch['url'] = 'https://example.com/doc.pdf'
assert watch.get_fetch_backend == 'html_requests'
watch['url'] = 'https://example.com'
# A group override beats the watch's own selection
tag_uuid = datastore.add_tag("Force webdriver")
client.post(url_for("tags.form_tag_edit_submit", uuid=tag_uuid),
data={'title': 'Force webdriver', 'browser_config_overrides_watch': 'y',
'browser_config': 'html_webdriver'}, follow_redirects=True)
watch['fetch_backend'] = 'html_requests'
watch['tags'] = [tag_uuid]
assert watch.get_fetch_backend == 'html_webdriver'
def test_update_34_normalises_default_browser(client, live_server, measure_memory_usage, datastore_path):
"""update_34 turns a blank/'system'/dangling global default into a concrete built-in engine,
but leaves an already-valid default (engine name or saved config id) untouched."""
datastore = client.application.config.get('DATASTORE')
app = datastore.data['settings']['application']
# Blank -> concrete default
app['fetch_backend'] = ''
datastore.update_34()
assert app['fetch_backend'] in ('html_requests', 'html_webdriver')
# 'system' at the global level is invalid (it IS the system default) -> normalised
app['fetch_backend'] = 'system'
datastore.update_34()
assert app['fetch_backend'] != 'system'
# A dangling browser-config id -> normalised to a concrete engine
app['fetch_backend'] = 'deleted-id-9999'
datastore.update_34()
assert app['fetch_backend'] != 'deleted-id-9999'
# A valid built-in engine is left untouched (idempotent)
app['fetch_backend'] = 'html_webdriver'
datastore.update_34()
assert app['fetch_backend'] == 'html_webdriver'
# A valid saved browser-config id is left untouched
_add_browser(client, label="Keeper", viewport_width=800)
cid = list(datastore.browser_config_store.all())[0]
app['fetch_backend'] = cid
datastore.update_34()
assert app['fetch_backend'] == cid
def test_group_override_with_builtin_browser(client, live_server, measure_memory_usage, datastore_path):
"""A group can also override with a built-in engine (e.g. html_webdriver), not just a user browser."""
from changedetectionio.model.browser_config import resolve_browser_config_override