From a89c30f8823446924e7953e8e95c516e01455ee6 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sat, 7 Feb 2026 03:41:02 +0100 Subject: [PATCH] adding notes --- changedetectionio/model/Tag.py | 37 +++++++ changedetectionio/model/Watch.py | 152 +++++++++++++++++++++++++++- changedetectionio/model/__init__.py | 22 +++- 3 files changed, 207 insertions(+), 4 deletions(-) diff --git a/changedetectionio/model/Tag.py b/changedetectionio/model/Tag.py index 34ba41452..f94b69650 100644 --- a/changedetectionio/model/Tag.py +++ b/changedetectionio/model/Tag.py @@ -1,8 +1,45 @@ +""" +Tag/Group domain model for organizing and overriding watch settings. + +ARCHITECTURE NOTE: Configuration Override Hierarchy +=================================================== + +Tags can override Watch settings when overrides_watch=True. +Current implementation requires manual checking in processors: + + for tag_uuid in watch.get('tags'): + tag = datastore['settings']['application']['tags'][tag_uuid] + if tag.get('overrides_watch'): + restock_settings = tag.get('restock_settings', {}) + break + +With Pydantic, this would be automatic via chain resolution: + Watch → Tag (first with overrides_watch) → Global + +See: Watch.py model docstring for full Pydantic architecture explanation +See: processors/restock_diff/processor.py:184-192 for current manual implementation +""" from changedetectionio.model import watch_base class model(watch_base): + """ + Tag domain model - groups watches and can override their settings. + + Tags inherit from watch_base to reuse all the same fields as Watch. + When overrides_watch=True, tag settings take precedence over watch settings + for all watches in this tag/group. + + Fields: + overrides_watch (bool): If True, this tag's settings override watch settings + title (str): Display name for this tag/group + uuid (str): Unique identifier + ... (all fields from watch_base can be set as tag-level overrides) + + Resolution order when overrides_watch=True: + Watch.field → Tag.field (if overrides_watch) → Global.field + """ def __init__(self, *arg, **kw): # Store datastore reference (optional for Tags, but good for consistency) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 7dd83e518..90f72c777 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -1,3 +1,29 @@ +""" +Watch domain model for change detection monitoring. + +ARCHITECTURE NOTE: Configuration Override Hierarchy +=================================================== + +This module implements Watch objects that inherit from dict (technical debt). +The dream architecture would use Pydantic for: + +1. CHAIN RESOLUTION (Watch → Tag → Global Settings) + - Current: Manual resolution scattered across codebase + - Future: @computed_field properties with automatic resolution + - Examples: resolved_fetch_backend, resolved_restock_settings, etc. + +2. DATABASE BACKEND ABSTRACTION + - Current: Domain model tightly coupled to file-based JSON storage + - Future: Domain model (Pydantic) separate from persistence layer + - Enables: Easy migration to PostgreSQL, MongoDB, etc. + +3. TYPE SAFETY & VALIDATION + - Current: Dict access with no compile-time checks + - Future: Type hints, IDE autocomplete, validation at boundaries + +See class model docstring for detailed explanation and examples. +See: processors/restock_diff/processor.py:184-192 for manual resolution example +""" import gc from copy import copy @@ -104,6 +130,99 @@ def _brotli_save(contents, filepath, mode=None, fallback_uncompressed=False): class model(watch_base): + """ + Watch domain model for monitoring URL changes. + + Inherits from watch_base (which inherits dict) - see watch_base docstring for field documentation. + + ## Configuration Override Hierarchy (Chain Resolution) + + The dream architecture uses a 3-level resolution chain: + Watch settings → Tag/Group settings → Global settings + + Current implementation is MANUAL (see processor.py:184-192 for example): + - Processors manually check watch.get('field') + - Then loop through watch.tags to find first tag with overrides_watch=True + - Finally fall back to datastore['settings']['application']['field'] + + FUTURE: Pydantic-based chain resolution would enable: + + ```python + # Instead of manual resolution in every processor: + restock_settings = watch.get('restock_settings', {}) + for tag_uuid in watch.get('tags'): + tag = datastore['settings']['application']['tags'][tag_uuid] + if tag.get('overrides_watch'): + restock_settings = tag.get('restock_settings', {}) + break + + # Clean computed properties with automatic resolution: + @computed_field + def resolved_restock_settings(self) -> dict: + if self.restock_settings: + return self.restock_settings + for tag_uuid in self.tags: + tag = self._datastore.get_tag(tag_uuid) + if tag.overrides_watch and tag.restock_settings: + return tag.restock_settings + return self._datastore.settings.restock_settings or {} + + # Usage: watch.resolved_restock_settings (automatic, type-safe, tested once) + ``` + + Benefits of Pydantic migration: + 1. Single source of truth for resolution logic (not scattered across processors) + 2. Type safety + IDE autocomplete (watch.resolved_fetch_backend vs dict navigation) + 3. Database backend abstraction (domain model separate from persistence) + 4. Automatic validation at boundaries + 5. Self-documenting via type hints + 6. Easy to test resolution independently + + Resolution chain examples that would benefit: + - fetch_backend: watch → tag → global (see get_fetch_backend property) + - notification_urls: watch → tag → global + - time_between_check: watch → global (see threshold_seconds) + - restock_settings: watch → tag (see processors/restock_diff/processor.py:184-192) + - history_snapshot_max_length: watch → global (see save_history_blob:550-556) + - All processor_config_* settings could use tag overrides + + ## Database Backend Abstraction with Pydantic + + Current: Watch inherits dict, tightly coupled to file-based JSON storage + Future: Domain model (Watch) separate from persistence layer + + ```python + # Domain model (database-agnostic) + class Watch(BaseModel): + uuid: str + url: str + # ... validation, business logic + + # Pluggable backends + class DataStoreBackend(ABC): + def save_watch(self, watch: Watch): ... + def load_watch(self, uuid: str) -> Watch: ... + + # Implementations: FileBackend, MongoBackend, PostgresBackend, etc. + ``` + + This would enable: + - Easy migration between storage backends (file → postgres → mongodb) + - Pydantic handles serialization/deserialization automatically + - Domain logic stays clean (no storage concerns in Watch methods) + + ## Migration Path + + Given existing codebase, incremental migration recommended: + 1. Create Pydantic models alongside existing dict-based models + 2. Add .to_pydantic() / .from_pydantic() bridge methods + 3. Gradually migrate code to use Pydantic models + 4. Remove dict inheritance once migration complete + + See: watch_base docstring for technical debt discussion + See: processors/restock_diff/processor.py:184-192 for manual resolution example + See: Watch.py:550-556 for nested dict navigation that would become watch.resolved_* + """ __newest_history_key = None __history_n = 0 jitter_seconds = 0 @@ -243,8 +362,30 @@ class model(watch_base): @property def get_fetch_backend(self): """ - Like just using the `fetch_backend` key but there could be some logic - :return: + Get the fetch backend for this watch with special case handling. + + CHAIN RESOLUTION OPPORTUNITY: + Currently returns watch.fetch_backend directly, but doesn't implement + Watch → Tag → Global resolution chain. With Pydantic: + + @computed_field + def resolved_fetch_backend(self) -> str: + # Special case: PDFs always use html_requests + if self.is_pdf: + return 'html_requests' + + # Watch override + if self.fetch_backend and self.fetch_backend != 'system': + return self.fetch_backend + + # Tag override (first tag with overrides_watch=True wins) + for tag_uuid in self.tags: + tag = self._datastore.get_tag(tag_uuid) + if tag.overrides_watch and tag.fetch_backend: + return tag.fetch_backend + + # Global default + return self._datastore.settings.fetch_backend """ # Maybe also if is_image etc? # This is because chrome/playwright wont render the PDF in the browser and we will just fetch it and use pdf2html to see the text. @@ -546,7 +687,12 @@ class model(watch_base): self.__newest_history_key = timestamp self.__history_n += 1 - + # MANUAL CHAIN RESOLUTION: Watch → Global + # With Pydantic, this would become: maxlen = watch.resolved_history_snapshot_max_length + # @computed_field def resolved_history_snapshot_max_length(self) -> Optional[int]: + # if self.history_snapshot_max_length: return self.history_snapshot_max_length + # if tag := self._get_override_tag(): return tag.history_snapshot_max_length + # return self._datastore.settings.history_snapshot_max_length maxlen = ( self.get('history_snapshot_max_length') or (self.__datastore and self.__datastore['settings']['application'].get('history_snapshot_max_length')) diff --git a/changedetectionio/model/__init__.py b/changedetectionio/model/__init__.py index ce22d598c..5aeea1ae8 100644 --- a/changedetectionio/model/__init__.py +++ b/changedetectionio/model/__init__.py @@ -13,13 +13,33 @@ class watch_base(dict): Dict inheritance is legacy technical debt that should be refactored to a proper domain model (e.g., Pydantic BaseModel) for better type safety and validation. - TODO: Migrate to Pydantic BaseModel or dataclass for: + TODO: Migrate to Pydantic BaseModel for: - Type safety and IDE autocomplete - Automatic validation - Clear separation between domain model and serialization + - Database backend abstraction (file → postgres → mongodb) + - Configuration override chain resolution (Watch → Tag → Global) - Immutability options - Better testing + CHAIN RESOLUTION ARCHITECTURE: + The dream is a 3-level override hierarchy: + Watch settings → Tag/Group settings → Global settings + + Current implementation: MANUAL resolution scattered across codebase + - Processors manually check watch.get('field') + - Loop through tags to find overrides_watch=True + - Fall back to datastore['settings']['application']['field'] + + Pydantic implementation: AUTOMATIC resolution via @computed_field + - Single source of truth for each setting's resolution logic + - Type-safe, testable, self-documenting + - Example: watch.resolved_fetch_backend (instead of nested dict navigation) + + See: Watch.py model docstring for detailed Pydantic architecture plan + See: Tag.py model docstring for tag override explanation + See: processors/restock_diff/processor.py:184-192 for current manual example + Core Fields: uuid (str): Unique identifier for this watch (auto-generated) url (str): Target URL to monitor for changes