diff --git a/changedetectionio/api/Tags.py b/changedetectionio/api/Tags.py
index 33791e346..0fdeb3aef 100644
--- a/changedetectionio/api/Tags.py
+++ b/changedetectionio/api/Tags.py
@@ -79,7 +79,7 @@ class Tag(Resource):
'browser_steps_last_error_step', 'check_count', 'consecutive_filter_failures',
'content-type', 'fetch_time', 'last_changed', 'last_checked', 'last_error',
'last_notification_error', 'last_viewed', 'notification_alert_count',
- 'page_title', 'previous_md5', 'previous_md5_before_filters', 'remote_server_reply'
+ 'page_title', 'previous_md5', 'remote_server_reply'
}
# Create clean tag dict without Watch-specific fields
@@ -160,6 +160,11 @@ class Tag(Resource):
tag.update(json_data)
tag.commit()
+ # Clear checksums for all watches using this tag to force reprocessing
+ # Tag changes affect inherited configuration
+ cleared_count = self.datastore.clear_checksums_for_tag(uuid)
+ logger.info(f"Tag {uuid} updated via API, cleared {cleared_count} watch checksums")
+
return "OK", 200
diff --git a/changedetectionio/api/__init__.py b/changedetectionio/api/__init__.py
index e0d9cc95a..f3b698845 100644
--- a/changedetectionio/api/__init__.py
+++ b/changedetectionio/api/__init__.py
@@ -70,46 +70,6 @@ def _resolve_schema_properties(schema_name):
return properties
-@functools.cache
-def _resolve_readonly_fields(schema_name):
- """
- Generic helper to resolve readOnly fields, including allOf inheritance.
-
- Args:
- schema_name: Name of the schema (e.g., 'Watch', 'Tag')
-
- Returns:
- frozenset: All readOnly field names including inherited ones
- """
- spec_dict = get_openapi_schema_dict()
- schema = spec_dict['components']['schemas'].get(schema_name, {})
-
- readonly_fields = set()
-
- # Handle allOf (schema inheritance)
- if 'allOf' in schema:
- for item in schema['allOf']:
- # Resolve $ref to parent schema
- if '$ref' in item:
- ref_path = item['$ref'].split('/')[-1]
- ref_schema = spec_dict['components']['schemas'].get(ref_path, {})
- if 'properties' in ref_schema:
- for field_name, field_def in ref_schema['properties'].items():
- if field_def.get('readOnly') is True:
- readonly_fields.add(field_name)
- # Check schema-specific properties
- if 'properties' in item:
- for field_name, field_def in item['properties'].items():
- if field_def.get('readOnly') is True:
- readonly_fields.add(field_name)
- else:
- # Direct properties (no inheritance)
- if 'properties' in schema:
- for field_name, field_def in schema['properties'].items():
- if field_def.get('readOnly') is True:
- readonly_fields.add(field_name)
-
- return frozenset(readonly_fields)
@functools.cache
def get_watch_schema_properties():
@@ -120,14 +80,8 @@ def get_watch_schema_properties():
"""
return _resolve_schema_properties('WatchBase')
-@functools.cache
-def get_readonly_watch_fields():
- """
- Extract readOnly field names from Watch schema in OpenAPI spec.
-
- Returns readOnly fields from WatchBase (uuid, date_created) + Watch-specific readOnly fields.
- """
- return _resolve_readonly_fields('Watch')
+# Import readonly field utilities from shared module (avoids circular dependencies with model layer)
+from changedetectionio.model.schema_utils import get_readonly_watch_fields, get_readonly_tag_fields
@functools.cache
def get_tag_schema_properties():
@@ -138,15 +92,6 @@ def get_tag_schema_properties():
"""
return _resolve_schema_properties('Tag')
-@functools.cache
-def get_readonly_tag_fields():
- """
- Extract readOnly field names from Tag schema in OpenAPI spec.
-
- Returns readOnly fields from WatchBase (uuid, date_created) + Tag-specific readOnly fields.
- """
- return _resolve_readonly_fields('Tag')
-
def validate_openapi_request(operation_id):
"""Decorator to validate incoming requests against OpenAPI spec."""
def decorator(f):
diff --git a/changedetectionio/blueprint/settings/__init__.py b/changedetectionio/blueprint/settings/__init__.py
index 03eb9f1f9..533892137 100644
--- a/changedetectionio/blueprint/settings/__init__.py
+++ b/changedetectionio/blueprint/settings/__init__.py
@@ -83,6 +83,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
datastore.data['settings']['requests'].update(form.data['requests'])
datastore.commit()
+ # Clear all checksums to force reprocessing with new settings
+ # Global settings can affect watch behavior (filters, rendering, etc.)
+ datastore.clear_all_last_checksums()
+
# Adjust worker count if it changed
if new_worker_count != old_worker_count:
from changedetectionio import worker_pool
diff --git a/changedetectionio/blueprint/tags/__init__.py b/changedetectionio/blueprint/tags/__init__.py
index 3d3a16255..abc3f79b4 100644
--- a/changedetectionio/blueprint/tags/__init__.py
+++ b/changedetectionio/blueprint/tags/__init__.py
@@ -244,6 +244,12 @@ def construct_blueprint(datastore: ChangeDetectionStore):
tag.update(form.data)
tag['processor'] = 'restock_diff'
tag.commit()
+
+ # Clear checksums for all watches using this tag to force reprocessing
+ # Tag changes affect inherited configuration
+ cleared_count = datastore.clear_checksums_for_tag(uuid)
+ logger.info(f"Tag {uuid} updated, cleared {cleared_count} watch checksums")
+
flash(gettext("Updated"))
return redirect(url_for('tags.tags_overview_page'))
diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py
index b4d509854..129ae651d 100644
--- a/changedetectionio/model/Watch.py
+++ b/changedetectionio/model/Watch.py
@@ -335,7 +335,6 @@ class model(EntityPersistenceMixin, watch_base):
'last_notification_error': False,
'last_viewed': 0,
'previous_md5': False,
- 'previous_md5_before_filters': False,
'remote_server_reply': None,
'track_ldjson_price_data': None
})
@@ -389,7 +388,7 @@ class model(EntityPersistenceMixin, watch_base):
# content_type field is set in the future
# https://github.com/dgtlmoon/changedetection.io/issues/1392
# Not sure the best logic here
- return self.get('url', '').lower().endswith('.pdf') or 'pdf' in self.get('content_type', '').lower()
+ return self.get('url', '').lower().endswith('.pdf') or 'pdf' in self.get('content-type', '').lower()
@property
def label(self):
diff --git a/changedetectionio/model/__init__.py b/changedetectionio/model/__init__.py
index c3192bf60..abd3e513a 100644
--- a/changedetectionio/model/__init__.py
+++ b/changedetectionio/model/__init__.py
@@ -129,7 +129,6 @@ class watch_base(dict):
fetch_time (float): Duration of last fetch in seconds
consecutive_filter_failures (int): Counter for consecutive filter match failures
previous_md5 (str|bool): MD5 hash of previous content
- previous_md5_before_filters (str|bool): MD5 hash before filters applied
history_snapshot_max_length (int|None): Max history snapshots to keep (None = use global)
Conditions:
@@ -166,6 +165,10 @@ class watch_base(dict):
if kw.get('datastore_path'):
del kw['datastore_path']
+ # IMPORTANT: Don't initialize __watch_was_edited yet!
+ # We'll initialize it AFTER the initial update() call below
+ # This prevents marking the watch as edited during initialization
+
self.update({
# Custom notification content
# Re #110, so then if this is set to None, we know to use the default value instead
@@ -211,7 +214,6 @@ class watch_base(dict):
'page_title': None, #
from the page
'paused': False,
'previous_md5': False,
- 'previous_md5_before_filters': False, # Used for skipping changedetection entirely
'processor': 'text_json_diff', # could be restock_diff or others from .processors
'price_change_threshold_percent': None,
'proxy': None, # Preferred proxy connection
@@ -297,9 +299,116 @@ class watch_base(dict):
super(watch_base, self).__init__(*arg, **kw)
+ # Check if we're being initialized from an existing watch object
+ # that has was_edited=True, so we can preserve the flag
+ preserve_edited_flag = False
if self.get('default'):
+ # When creating a new watch object from an existing one (e.g., changing processor),
+ # preserve the was_edited flag if it was True
+ default_watch = self.get('default')
+ if hasattr(default_watch, 'was_edited') and default_watch.was_edited:
+ preserve_edited_flag = True
del self['default']
+ # NOW initialize the edited flag after all initial setup is complete
+ # This ensures initialization doesn't trigger the edited flag
+ # But preserve it if the source watch had it set to True
+ self.__watch_was_edited = preserve_edited_flag
+
+ def _mark_field_as_edited(self, key):
+ """
+ Helper to mark a field as edited if it's writable.
+
+ Internal method used by __setitem__, update(), pop(), etc.
+ """
+ # Don't track edits during initial load or if already edited
+ if not hasattr(self, '_watch_base__watch_was_edited'):
+ return
+ if self.__watch_was_edited:
+ return # Already marked as edited
+
+ # Import from shared schema utilities (no circular dependency)
+ from .schema_utils import get_readonly_watch_fields
+ readonly_fields = get_readonly_watch_fields()
+
+ # Additional system-managed fields not in OpenAPI spec (yet)
+ # These are set by processors/workers and should not trigger edited flag
+ additional_system_fields = {
+ 'last_check_status', # Set by processors
+ 'restock', # Set by restock processor
+ 'last_viewed', # Set by mark_all_viewed endpoint
+ }
+
+ # Only mark as edited if this is a user-writable field
+ if key not in readonly_fields and key not in additional_system_fields:
+ self.__watch_was_edited = True
+
+ def __setitem__(self, key, value):
+ """
+ Override dict.__setitem__ to track when writable watch fields are modified.
+
+ This enables skipping reprocessing when:
+ 1. HTML content is unchanged (checksumFromPreviousCheckWasTheSame)
+ 2. AND watch configuration was not edited
+
+ Only sets the edited flag when field is NOT in readonly_fields (from OpenAPI spec).
+ """
+ # Set the value first (always)
+ super().__setitem__(key, value)
+ # Mark as edited if writable field
+ self._mark_field_as_edited(key)
+
+ def __delitem__(self, key):
+ """Override dict.__delitem__ to track deletions of writable fields."""
+ super().__delitem__(key)
+ self._mark_field_as_edited(key)
+
+ def update(self, *args, **kwargs):
+ """Override dict.update() to track modifications to writable fields."""
+ # Call parent update first
+ super().update(*args, **kwargs)
+
+ # Mark as edited for any writable fields that were updated
+ # Handle both update(dict) and update(key=value) forms
+ if args:
+ for key in args[0].keys():
+ self._mark_field_as_edited(key)
+ for key in kwargs.keys():
+ self._mark_field_as_edited(key)
+
+ def pop(self, key, *args):
+ """Override dict.pop() to track removal of writable fields."""
+ result = super().pop(key, *args)
+ self._mark_field_as_edited(key)
+ return result
+
+ def setdefault(self, key, default=None):
+ """Override dict.setdefault() to track modifications to writable fields."""
+ # Only marks as edited if key didn't exist (i.e., a new value was set)
+ existed = key in self
+ result = super().setdefault(key, default)
+ if not existed:
+ self._mark_field_as_edited(key)
+ return result
+
+ @property
+ def was_edited(self):
+ """
+ Check if watch configuration was edited since last processing.
+
+ Returns:
+ bool: True if writable fields were modified, False otherwise
+ """
+ return getattr(self, '_watch_base__watch_was_edited', False)
+
+ def reset_watch_edited_flag(self):
+ """
+ Reset the watch edited flag after successful processing.
+
+ Call this after processing completes to allow future content-only change detection.
+ """
+ self.__watch_was_edited = False
+
@classmethod
def get_property_names(cls):
"""
diff --git a/changedetectionio/model/schema_utils.py b/changedetectionio/model/schema_utils.py
new file mode 100644
index 000000000..d30ffcfbb
--- /dev/null
+++ b/changedetectionio/model/schema_utils.py
@@ -0,0 +1,92 @@
+"""
+Schema utilities for Watch and Tag models.
+
+Provides functions to extract readonly fields and properties from OpenAPI spec.
+Shared by both the model layer and API layer to avoid circular dependencies.
+"""
+
+import functools
+
+
+@functools.cache
+def get_openapi_schema_dict():
+ """
+ Get the raw OpenAPI spec dictionary for schema access.
+
+ Returns the YAML dict directly (not the OpenAPI object).
+ """
+ import os
+ import yaml
+
+ spec_path = os.path.join(os.path.dirname(__file__), '../../docs/api-spec.yaml')
+ if not os.path.exists(spec_path):
+ spec_path = os.path.join(os.path.dirname(__file__), '../docs/api-spec.yaml')
+
+ with open(spec_path, 'r', encoding='utf-8') as f:
+ return yaml.safe_load(f)
+
+
+@functools.cache
+def _resolve_readonly_fields(schema_name):
+ """
+ Generic helper to resolve readOnly fields, including allOf inheritance.
+
+ Args:
+ schema_name: Name of the schema (e.g., 'Watch', 'Tag')
+
+ Returns:
+ frozenset: All readOnly field names including inherited ones
+ """
+ spec_dict = get_openapi_schema_dict()
+ schema = spec_dict['components']['schemas'].get(schema_name, {})
+
+ readonly_fields = set()
+
+ # Handle allOf (schema inheritance)
+ if 'allOf' in schema:
+ for item in schema['allOf']:
+ # Resolve $ref to parent schema
+ if '$ref' in item:
+ ref_path = item['$ref'].split('/')[-1]
+ ref_schema = spec_dict['components']['schemas'].get(ref_path, {})
+ if 'properties' in ref_schema:
+ for field_name, field_def in ref_schema['properties'].items():
+ if field_def.get('readOnly') is True:
+ readonly_fields.add(field_name)
+ # Check schema-specific properties
+ if 'properties' in item:
+ for field_name, field_def in item['properties'].items():
+ if field_def.get('readOnly') is True:
+ readonly_fields.add(field_name)
+ else:
+ # Direct properties (no inheritance)
+ if 'properties' in schema:
+ for field_name, field_def in schema['properties'].items():
+ if field_def.get('readOnly') is True:
+ readonly_fields.add(field_name)
+
+ return frozenset(readonly_fields)
+
+
+@functools.cache
+def get_readonly_watch_fields():
+ """
+ Extract readOnly field names from Watch schema in OpenAPI spec.
+
+ Returns readOnly fields from WatchBase (uuid, date_created) + Watch-specific readOnly fields.
+
+ Used by:
+ - model/watch_base.py: Track when writable fields are edited
+ - api/Watch.py: Filter readonly fields from PUT requests
+ """
+ return _resolve_readonly_fields('Watch')
+
+
+@functools.cache
+def get_readonly_tag_fields():
+ """
+ Extract readOnly field names from Tag schema in OpenAPI spec.
+
+ Returns readOnly fields from WatchBase (uuid, date_created) + Tag-specific readOnly fields.
+ """
+ return _resolve_readonly_fields('Tag')
diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py
index 5fedc0b53..fbb448aa4 100644
--- a/changedetectionio/processors/__init__.py
+++ b/changedetectionio/processors/__init__.py
@@ -1,6 +1,6 @@
from functools import lru_cache
from loguru import logger
-from flask_babel import gettext
+from flask_babel import gettext, get_locale
import importlib
import inspect
import os
@@ -190,14 +190,15 @@ def get_plugin_processor_metadata():
logger.warning(f"Error getting plugin processor metadata: {e}")
return metadata
-
-def available_processors():
- """
- Get a list of processors by name and description for the UI elements.
- Can be filtered via DISABLED_PROCESSORS environment variable (comma-separated list).
- :return: A list :)
+@lru_cache(maxsize=32)
+def _available_processors_cached(locale_str):
"""
+ Internal cached function that includes locale in cache key.
+ This ensures translations are cached per-language instead of globally.
+ :param locale_str: The locale string (e.g., 'en', 'it', 'zh')
+ :return: A list of tuples (processor_name, translated_description, weight)
+ """
processor_classes = find_processors()
# Check if DISABLED_PROCESSORS env var is set
@@ -256,6 +257,22 @@ def available_processors():
# Return as tuples without weight (for backwards compatibility)
return [(name, desc) for name, desc, weight in available]
+def available_processors():
+ """
+ Get a list of processors by name and description for the UI elements.
+ Can be filtered via DISABLED_PROCESSORS environment variable (comma-separated list).
+
+ This function delegates to a locale-aware cached version to ensure translations
+ are cached per-language instead of globally.
+
+ :return: A list of tuples (processor_name, translated_description)
+ """
+ # Get current locale and use it as cache key
+ # Convert Babel Locale object to string for use as cache key
+ locale = get_locale()
+ locale_str = str(locale) if locale else 'en'
+ return _available_processors_cached(locale_str)
+
def get_default_processor():
"""
diff --git a/changedetectionio/processors/base.py b/changedetectionio/processors/base.py
index 99e306fd9..67b54e2ac 100644
--- a/changedetectionio/processors/base.py
+++ b/changedetectionio/processors/base.py
@@ -19,6 +19,7 @@ class difference_detection_processor():
xpath_data = None
preferred_proxy = None
screenshot_format = SCREENSHOT_FORMAT_JPEG
+ last_raw_content_checksum = None
def __init__(self, datastore, watch_uuid):
self.datastore = datastore
@@ -34,6 +35,64 @@ class difference_detection_processor():
# Generic fetcher that should be extended (requests, playwright etc)
self.fetcher = Fetcher()
+ # Load the last raw content checksum from file
+ self.read_last_raw_content_checksum()
+
+ def update_last_raw_content_checksum(self, checksum):
+ """
+ Save the raw content MD5 checksum to file.
+ This is used for skip logic - avoid reprocessing if raw HTML unchanged.
+ """
+ if not checksum:
+ return
+
+ watch = self.datastore.data['watching'].get(self.watch_uuid)
+ if not watch:
+ return
+
+ data_dir = watch.data_dir
+ if not data_dir:
+ return
+
+ watch.ensure_data_dir_exists()
+ checksum_file = os.path.join(data_dir, 'last-checksum.txt')
+
+ try:
+ with open(checksum_file, 'w', encoding='utf-8') as f:
+ f.write(checksum)
+ self.last_raw_content_checksum = checksum
+ except IOError as e:
+ logger.warning(f"Failed to write checksum file for {self.watch_uuid}: {e}")
+
+ def read_last_raw_content_checksum(self):
+ """
+ Read the last raw content MD5 checksum from file.
+ Returns None if file doesn't exist (first run) or can't be read.
+ """
+ watch = self.datastore.data['watching'].get(self.watch_uuid)
+ if not watch:
+ self.last_raw_content_checksum = None
+ return
+
+ data_dir = watch.data_dir
+ if not data_dir:
+ self.last_raw_content_checksum = None
+ return
+
+ checksum_file = os.path.join(data_dir, 'last-checksum.txt')
+
+ if not os.path.isfile(checksum_file):
+ self.last_raw_content_checksum = None
+ return
+
+ try:
+ with open(checksum_file, 'r', encoding='utf-8') as f:
+ self.last_raw_content_checksum = f.read().strip()
+ except IOError as e:
+ logger.warning(f"Failed to read checksum file for {self.watch_uuid}: {e}")
+ self.last_raw_content_checksum = None
+
+
async def call_browser(self, preferred_proxy_id=None):
from requests.structures import CaseInsensitiveDict
@@ -257,8 +316,16 @@ class difference_detection_processor():
except IOError as e:
logger.error(f"Failed to write extra watch config {filename}: {e}")
+ def get_raw_document_checksum(self):
+ checksum = None
+
+ if self.fetcher.content:
+ checksum = hashlib.md5(self.fetcher.content.encode('utf-8')).hexdigest()
+
+ return checksum
+
@abstractmethod
- def run_changedetection(self, watch):
+ def run_changedetection(self, watch, force_reprocess=False):
update_obj = {'last_notification_error': False, 'last_error': False}
some_data = 'xxxxx'
update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()
diff --git a/changedetectionio/processors/image_ssim_diff/processor.py b/changedetectionio/processors/image_ssim_diff/processor.py
index 17fab1a8c..422ca1aab 100644
--- a/changedetectionio/processors/image_ssim_diff/processor.py
+++ b/changedetectionio/processors/image_ssim_diff/processor.py
@@ -30,7 +30,7 @@ class perform_site_check(difference_detection_processor):
# Override to use PNG format for better image comparison (JPEG compression creates noise)
screenshot_format = SCREENSHOT_FORMAT_PNG
- def run_changedetection(self, watch):
+ def run_changedetection(self, watch, force_reprocess=False):
"""
Perform screenshot comparison using OpenCV subprocess handler.
diff --git a/changedetectionio/processors/restock_diff/processor.py b/changedetectionio/processors/restock_diff/processor.py
index ba89b4e3b..0665b43d8 100644
--- a/changedetectionio/processors/restock_diff/processor.py
+++ b/changedetectionio/processors/restock_diff/processor.py
@@ -2,6 +2,7 @@ from ..base import difference_detection_processor
from ..exceptions import ProcessorException
from . import Restock
from loguru import logger
+from changedetectionio.content_fetchers.exceptions import checksumFromPreviousCheckWasTheSame
import urllib3
import time
@@ -403,22 +404,37 @@ class perform_site_check(difference_detection_processor):
screenshot = None
xpath_data = None
- def run_changedetection(self, watch):
+ def run_changedetection(self, watch, force_reprocess=False):
import hashlib
if not watch:
raise Exception("Watch no longer exists.")
+ current_raw_document_checksum = self.get_raw_document_checksum()
+ # Skip processing only if BOTH conditions are true:
+ # 1. HTML content unchanged (checksum matches last saved checksum)
+ # 2. Watch configuration was not edited (including trigger_text, filters, etc.)
+ # The was_edited flag handles all watch configuration changes, so we don't need
+ # separate checks for trigger_text or other processing rules.
+ if (not force_reprocess and
+ not watch.was_edited and
+ self.last_raw_content_checksum and
+ self.last_raw_content_checksum == current_raw_document_checksum):
+ raise checksumFromPreviousCheckWasTheSame()
+
# Unset any existing notification error
update_obj = {'last_notification_error': False, 'last_error': False, 'restock': Restock()}
self.screenshot = self.fetcher.screenshot
self.xpath_data = self.fetcher.xpath_data
- # Track the content type
- update_obj['content_type'] = self.fetcher.headers.get('Content-Type', '')
+ # Track the content type (readonly field, doesn't trigger was_edited)
+ update_obj['content-type'] = self.fetcher.headers.get('Content-Type', '') # Use hyphen (matches OpenAPI spec)
update_obj["last_check_status"] = self.fetcher.get_last_status_code()
+ # Save the raw content checksum to file (processor implementation detail, not watch config)
+ self.update_last_raw_content_checksum(current_raw_document_checksum)
+
# Only try to process restock information (like scraping for keywords) if the page was actually rendered correctly.
# Otherwise it will assume "in stock" because nothing suggesting the opposite was found
from ...html_tools import html_to_text
diff --git a/changedetectionio/processors/text_json_diff/__init__.py b/changedetectionio/processors/text_json_diff/__init__.py
index c1c9d8c86..52dcbbd6d 100644
--- a/changedetectionio/processors/text_json_diff/__init__.py
+++ b/changedetectionio/processors/text_json_diff/__init__.py
@@ -17,7 +17,8 @@ def _task(watch, update_handler):
try:
# The slow process (we run 2 of these in parallel)
- changed_detected, update_obj, text_after_filter = update_handler.run_changedetection(watch=watch)
+ # Always force reprocess for preview - we want to show the filtered content regardless of checksums
+ changed_detected, update_obj, text_after_filter = update_handler.run_changedetection(watch=watch, force_reprocess=True)
except FilterNotFoundInResponse as e:
text_after_filter = f"Filter not found in HTML: {str(e)}"
except ReplyWithContentButNoText as e:
diff --git a/changedetectionio/processors/text_json_diff/processor.py b/changedetectionio/processors/text_json_diff/processor.py
index 8f999944d..b039723e3 100644
--- a/changedetectionio/processors/text_json_diff/processor.py
+++ b/changedetectionio/processors/text_json_diff/processor.py
@@ -7,6 +7,7 @@ import re
import urllib3
from changedetectionio.conditions import execute_ruleset_against_all_plugins
+from changedetectionio.content_fetchers.exceptions import checksumFromPreviousCheckWasTheSame
from ..base import difference_detection_processor
from changedetectionio.html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text, TRANSLATE_WHITESPACE_TABLE
from changedetectionio import html_tools, content_fetchers
@@ -368,12 +369,24 @@ class ChecksumCalculator:
# (set_proxy_from_list)
class perform_site_check(difference_detection_processor):
- def run_changedetection(self, watch):
+ def run_changedetection(self, watch, force_reprocess=False):
changed_detected = False
if not watch:
raise Exception("Watch no longer exists.")
+ current_raw_document_checksum = self.get_raw_document_checksum()
+ # Skip processing only if BOTH conditions are true:
+ # 1. HTML content unchanged (checksum matches last saved checksum)
+ # 2. Watch configuration was not edited (including trigger_text, filters, etc.)
+ # The was_edited flag handles all watch configuration changes, so we don't need
+ # separate checks for trigger_text or other processing rules.
+ if (not force_reprocess and
+ not watch.was_edited and
+ self.last_raw_content_checksum and
+ self.last_raw_content_checksum == current_raw_document_checksum):
+ raise checksumFromPreviousCheckWasTheSame()
+
# Initialize components
filter_config = FilterConfig(watch, self.datastore)
content_processor = ContentProcessor(self.fetcher, watch, filter_config, self.datastore)
@@ -391,9 +404,11 @@ class perform_site_check(difference_detection_processor):
self.screenshot = self.fetcher.screenshot
self.xpath_data = self.fetcher.xpath_data
- # Track the content type and checksum before filters
- update_obj['content_type'] = ctype_header
- update_obj['previous_md5_before_filters'] = hashlib.md5(self.fetcher.content.encode('utf-8')).hexdigest()
+ # Track the content type (readonly field, doesn't trigger was_edited)
+ update_obj['content-type'] = ctype_header # Use hyphen (matches OpenAPI spec and watch_base default)
+
+ # Save the raw content checksum to file (processor implementation detail, not watch config)
+ self.update_last_raw_content_checksum(current_raw_document_checksum)
# === CONTENT PREPROCESSING ===
# Avoid creating unnecessary intermediate string copies by reassigning only when needed
diff --git a/changedetectionio/store/__init__.py b/changedetectionio/store/__init__.py
index bad4bf4e5..52383658c 100644
--- a/changedetectionio/store/__init__.py
+++ b/changedetectionio/store/__init__.py
@@ -456,6 +456,63 @@ class ChangeDetectionStore(DatastoreUpdatesMixin, FileSavingDataStore):
self.__data['settings']['application']['password'] = False
self.commit()
+ def clear_all_last_checksums(self):
+ """
+ Delete all last-checksum.txt files to force reprocessing of all watches.
+
+ This should be called when global settings change, since watches inherit
+ configuration and need to reprocess even if their individual watch dict
+ hasn't been modified.
+
+ Note: We delete the checksum file rather than setting was_edited=True because:
+ - was_edited is not persisted across restarts
+ - File deletion ensures reprocessing works across app restarts
+ """
+ deleted_count = 0
+ for uuid in self.__data['watching'].keys():
+ watch = self.__data['watching'][uuid]
+ if watch.data_dir:
+ checksum_file = os.path.join(watch.data_dir, 'last-checksum.txt')
+ if os.path.isfile(checksum_file):
+ try:
+ os.remove(checksum_file)
+ deleted_count += 1
+ logger.debug(f"Cleared checksum for watch {uuid}")
+ except OSError as e:
+ logger.warning(f"Failed to delete checksum file for {uuid}: {e}")
+
+ logger.info(f"Cleared {deleted_count} checksum files to force reprocessing")
+ return deleted_count
+
+ def clear_checksums_for_tag(self, tag_uuid):
+ """
+ Delete last-checksum.txt files for all watches using a specific tag.
+
+ This should be called when a tag configuration is edited, since watches
+ inherit tag settings and need to reprocess.
+
+ Args:
+ tag_uuid: UUID of the tag that was modified
+
+ Returns:
+ int: Number of checksum files deleted
+ """
+ deleted_count = 0
+ for uuid, watch in self.__data['watching'].items():
+ if watch.get('tags') and tag_uuid in watch['tags']:
+ if watch.data_dir:
+ checksum_file = os.path.join(watch.data_dir, 'last-checksum.txt')
+ if os.path.isfile(checksum_file):
+ try:
+ os.remove(checksum_file)
+ deleted_count += 1
+ logger.debug(f"Cleared checksum for watch {uuid} (tag {tag_uuid})")
+ except OSError as e:
+ logger.warning(f"Failed to delete checksum file for {uuid}: {e}")
+
+ logger.info(f"Cleared {deleted_count} checksum files for tag {tag_uuid}")
+ return deleted_count
+
def commit(self):
"""
Save settings immediately to disk using atomic write.
diff --git a/changedetectionio/tests/test_backup.py b/changedetectionio/tests/test_backup.py
index 526520199..449dda7e3 100644
--- a/changedetectionio/tests/test_backup.py
+++ b/changedetectionio/tests/test_backup.py
@@ -54,11 +54,11 @@ def test_backup(client, live_server, measure_memory_usage, datastore_path):
backup = ZipFile(io.BytesIO(res.data))
l = backup.namelist()
- # Check for UUID-based txt files (history and snapshot)
+ # Check for UUID-based txt files (history, snapshot, and last-checksum)
uuid4hex_txt = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}.*txt', re.I)
txt_files = list(filter(uuid4hex_txt.match, l))
- # Should be two txt files in the archive (history and the snapshot)
- assert len(txt_files) == 2
+ # Should be three txt files in the archive (history, snapshot, and last-checksum)
+ assert len(txt_files) == 3
# Check for watch.json files (new format)
uuid4hex_json = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}/watch\.json$', re.I)
diff --git a/changedetectionio/tests/test_css_selector.py b/changedetectionio/tests/test_css_selector.py
index 03914ec42..5a4f364d1 100644
--- a/changedetectionio/tests/test_css_selector.py
+++ b/changedetectionio/tests/test_css_selector.py
@@ -71,22 +71,19 @@ def test_include_filters_output():
# Tests the whole stack works with the CSS Filter
def test_check_markup_include_filters_restriction(client, live_server, measure_memory_usage, datastore_path):
- sleep_time_for_fetch_thread = 3
include_filters = "#sametext"
set_original_response(datastore_path=datastore_path)
- # Give the endpoint time to spin up
- time.sleep(1)
# Add our URL to the import page
test_url = url_for('test_endpoint', _external=True)
uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
- # Give the thread time to pick it up
- time.sleep(sleep_time_for_fetch_thread)
+ wait_for_all_checks(client)
+
# Goto the edit page, add our ignore text
# Add our URL to the import page
@@ -103,15 +100,15 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
)
assert bytes(include_filters.encode('utf-8')) in res.data
- # Give the thread time to pick it up
- time.sleep(sleep_time_for_fetch_thread)
+ wait_for_all_checks(client)
+
# Make a change
set_modified_response(datastore_path=datastore_path)
# Trigger a check
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
- # Give the thread time to pick it up
- time.sleep(sleep_time_for_fetch_thread)
+ wait_for_all_checks(client)
+
# It should have 'has-unread-changes' still
# Because it should be looking at only that 'sametext' id
diff --git a/changedetectionio/tests/test_history_consistency.py b/changedetectionio/tests/test_history_consistency.py
index 684ed2995..35bc17ff3 100644
--- a/changedetectionio/tests/test_history_consistency.py
+++ b/changedetectionio/tests/test_history_consistency.py
@@ -106,7 +106,7 @@ def test_consistent_history(client, live_server, measure_memory_usage, datastore
# Find the snapshot one
for fname in files_in_watch_dir:
- if fname != 'history.txt' and fname != 'watch.json' and 'html' not in fname:
+ if fname != 'history.txt' and fname != 'watch.json' and fname != 'last-checksum.txt' and 'html' not in fname:
if strtobool(os.getenv("TEST_WITH_BROTLI")):
assert fname.endswith('.br'), "Forced TEST_WITH_BROTLI then it should be a .br filename"
@@ -123,11 +123,18 @@ def test_consistent_history(client, live_server, measure_memory_usage, datastore
assert json_obj['watching'][w]['title'], "Watch should have a title set"
assert contents.startswith(watch_title + "x"), f"Snapshot contents in file {fname} should start with '{watch_title}x', got '{contents}'"
- # With new format, we also have watch.json, so 4 files total
+ # With new format, we have watch.json, so 4 files minimum
+ # Note: last-checksum.txt may or may not exist - it gets cleared by settings changes,
+ # and this test changes settings before checking files
+ # This assertion should be AFTER the loop, not inside it
if os.path.exists(changedetection_json):
- assert len(files_in_watch_dir) == 4, "Should be four files in the dir with new format: watch.json, html.br snapshot, history.txt and the extracted text snapshot"
+ # 4 required files: watch.json, html.br, history.txt, extracted text snapshot
+ # last-checksum.txt is optional (cleared by settings changes in this test)
+ assert len(files_in_watch_dir) >= 4 and len(files_in_watch_dir) <= 5, f"Should be 4-5 files in the dir with new format (last-checksum.txt is optional). Found {len(files_in_watch_dir)}: {files_in_watch_dir}"
else:
- assert len(files_in_watch_dir) == 3, "Should be just three files in the dir with legacy format: html.br snapshot, history.txt and the extracted text snapshot"
+ # 3 required files: html.br, history.txt, extracted text snapshot
+ # last-checksum.txt is optional
+ assert len(files_in_watch_dir) >= 3 and len(files_in_watch_dir) <= 4, f"Should be 3-4 files in the dir with legacy format (last-checksum.txt is optional). Found {len(files_in_watch_dir)}: {files_in_watch_dir}"
# Check that 'default' Watch vars aren't accidentally being saved
if os.path.exists(changedetection_json):
diff --git a/changedetectionio/tests/test_watch_edited_flag.py b/changedetectionio/tests/test_watch_edited_flag.py
new file mode 100644
index 000000000..459742d1f
--- /dev/null
+++ b/changedetectionio/tests/test_watch_edited_flag.py
@@ -0,0 +1,246 @@
+#!/usr/bin/env python3
+"""
+Test the watch edited flag functionality.
+
+This tests the private __watch_was_edited flag that tracks when writable
+watch fields are modified, which prevents skipping reprocessing when the
+watch configuration has changed.
+"""
+
+import os
+import time
+from flask import url_for
+from .util import live_server_setup, wait_for_all_checks
+
+
+def set_test_content(datastore_path):
+ """Write test HTML content to endpoint-content.txt for test server."""
+ test_html = """
+
+ Test content for watch edited flag tests
+ This content stays the same across checks
+
+
+ """
+ with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+ f.write(test_html)
+
+
+def test_watch_edited_flag_lifecycle(client, live_server, measure_memory_usage, datastore_path):
+ """
+ Test the full lifecycle of the was_edited flag:
+ 1. Flag starts False when watch is created
+ 2. Flag becomes True when writable fields are modified
+ 3. Flag is reset False after worker processing
+ 4. Flag stays False when readonly fields are modified
+ """
+
+ # Setup - Add a watch
+ test_url = url_for('test_endpoint', _external=True)
+ res = client.post(
+ url_for("ui.ui_views.form_quick_watch_add"),
+ data={"url": test_url, "tags": "", "edit_and_watch_submit_button": "Edit > Watch"},
+ follow_redirects=True
+ )
+ assert b"Watch added" in res.data or b"Updated watch" in res.data
+
+ # Get the watch UUID
+ datastore = client.application.config.get('DATASTORE')
+ uuid = list(datastore.data['watching'].keys())[0]
+ watch = datastore.data['watching'][uuid]
+
+ # Reset flag after initial form submission (form sets fields which trigger the flag)
+ watch.reset_watch_edited_flag()
+
+ # Test 1: Flag should be False after reset
+ assert not watch.was_edited, "Flag should be False after reset"
+
+ # Test 2: Modify a writable field (title) - flag should become True
+ watch['title'] = 'New Title'
+ assert watch.was_edited, "Flag should be True after modifying writable field 'title'"
+
+ # Test 3: Reset flag manually (simulating what worker does)
+ watch.reset_watch_edited_flag()
+ assert not watch.was_edited, "Flag should be False after reset"
+
+ # Test 4: Modify another writable field (url) - flag should become True again
+ watch['url'] = 'https://example.com'
+ assert watch.was_edited, "Flag should be True after modifying writable field 'url'"
+
+ # Test 5: Reset and modify a readonly field - flag should stay False
+ watch.reset_watch_edited_flag()
+ assert not watch.was_edited, "Flag should be False after reset"
+
+ # Modify readonly field (uuid) - should not set flag
+ old_uuid = watch['uuid']
+ watch['uuid'] = 'readonly-test-uuid'
+ assert not watch.was_edited, "Flag should stay False when modifying readonly field 'uuid'"
+ watch['uuid'] = old_uuid # Restore original
+
+ # Note: Worker reset behavior is tested in test_check_removed_line_contains_trigger
+ # and test_watch_edited_flag_prevents_skip
+
+ print("✓ All watch edited flag lifecycle tests passed")
+
+
+def test_watch_edited_flag_dict_methods(client, live_server, measure_memory_usage, datastore_path):
+ """
+ Test that the flag is set correctly by various dict methods:
+ - __setitem__ (watch['key'] = value)
+ - update() (watch.update({'key': value}))
+ - setdefault() (watch.setdefault('key', default))
+ - pop() (watch.pop('key'))
+ - __delitem__ (del watch['key'])
+ """
+
+ # Setup - Add a watch
+ test_url = url_for('test_endpoint', _external=True)
+ res = client.post(
+ url_for("ui.ui_views.form_quick_watch_add"),
+ data={"url": test_url, "tags": "", "edit_and_watch_submit_button": "Edit > Watch"},
+ follow_redirects=True
+ )
+
+ datastore = client.application.config.get('DATASTORE')
+ uuid = list(datastore.data['watching'].keys())[0]
+ watch = datastore.data['watching'][uuid]
+
+ # Test __setitem__
+ watch.reset_watch_edited_flag()
+ watch['title'] = 'Test via setitem'
+ assert watch.was_edited, "Flag should be True after __setitem__ on writable field"
+
+ # Test update() with dict
+ watch.reset_watch_edited_flag()
+ watch.update({'title': 'Test via update dict'})
+ assert watch.was_edited, "Flag should be True after update() with writable field"
+
+ # Test update() with kwargs
+ watch.reset_watch_edited_flag()
+ watch.update(title='Test via update kwargs')
+ assert watch.was_edited, "Flag should be True after update() kwargs with writable field"
+
+ # Test setdefault() on new key
+ watch.reset_watch_edited_flag()
+ watch.setdefault('title', 'Should not be set') # Key exists, no change
+ assert not watch.was_edited, "Flag should stay False when setdefault() doesn't change existing key"
+
+ watch.setdefault('custom_field', 'New value') # New key
+ assert watch.was_edited, "Flag should be True after setdefault() creates new writable field"
+
+ # Test pop() on writable field
+ watch.reset_watch_edited_flag()
+ watch.pop('custom_field', None)
+ assert watch.was_edited, "Flag should be True after pop() on writable field"
+
+ # Test __delitem__ on writable field
+ watch.reset_watch_edited_flag()
+ watch['temp_field'] = 'temp'
+ watch.reset_watch_edited_flag() # Reset after adding
+ del watch['temp_field']
+ assert watch.was_edited, "Flag should be True after __delitem__ on writable field"
+
+ print("✓ All dict methods correctly set the flag")
+
+
+def test_watch_edited_flag_prevents_skip(client, live_server, measure_memory_usage, datastore_path):
+ """
+ Test that the was_edited flag prevents skipping reprocessing.
+ When watch configuration is edited, it should reprocess even if content unchanged.
+ After worker processing, flag should be reset and subsequent checks can skip.
+ """
+
+ # Setup test content
+ set_test_content(datastore_path)
+
+ # Setup - Add a watch
+ test_url = url_for('test_endpoint', _external=True)
+ res = client.post(
+ url_for("ui.ui_views.form_quick_watch_add"),
+ data={"url": test_url, "tags": "", "edit_and_watch_submit_button": "Edit > Watch"},
+ follow_redirects=True
+ )
+ assert b"Watch added" in res.data or b"Updated watch" in res.data
+
+ datastore = client.application.config.get('DATASTORE')
+ uuid = list(datastore.data['watching'].keys())[0]
+ watch = datastore.data['watching'][uuid]
+
+ # Unpause the watch (watches are paused by default in tests)
+ watch['paused'] = False
+
+ # Run first check to establish baseline
+ client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+ wait_for_all_checks(client)
+
+ # Verify first check completed successfully - checksum file should exist
+ checksum_file = os.path.join(datastore_path, uuid, 'last-checksum.txt')
+ assert os.path.isfile(checksum_file), "First check should create last-checksum.txt file"
+
+ # Reset the was_edited flag (simulating clean state after processing)
+ watch.reset_watch_edited_flag()
+ assert not watch.was_edited, "Flag should be False after reset"
+
+ # Run second check without any changes - should skip via checksumFromPreviousCheckWasTheSame
+ client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+ wait_for_all_checks(client)
+
+ # Verify it was skipped (last_check_status should indicate skip)
+ # Note: The actual skip is tested in test_check_removed_line_contains_trigger
+ # Here we're focused on the was_edited flag interaction
+
+ # Now modify the watch - flag should become True
+ watch['title'] = 'Modified Title'
+ assert watch.was_edited, "Flag should be True after modifying watch"
+
+ # Run third check - should NOT skip because was_edited=True even though content unchanged
+ client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+ wait_for_all_checks(client)
+
+ # After worker processing, the flag should be reset by the worker
+ # This reset happens in the processor's run() method after processing completes
+ assert not watch.was_edited, "Flag should be False after worker processing"
+
+ print("✓ was_edited flag correctly prevents skip and is reset by worker")
+
+
+def test_watch_edited_flag_system_fields(client, live_server, measure_memory_usage, datastore_path):
+ """
+ Test that system fields (readonly + additional system fields) don't trigger the flag.
+ """
+
+ # Setup - Add a watch
+ test_url = url_for('test_endpoint', _external=True)
+ res = client.post(
+ url_for("ui.ui_views.form_quick_watch_add"),
+ data={"url": test_url, "tags": "", "edit_and_watch_submit_button": "Edit > Watch"},
+ follow_redirects=True
+ )
+
+ datastore = client.application.config.get('DATASTORE')
+ uuid = list(datastore.data['watching'].keys())[0]
+ watch = datastore.data['watching'][uuid]
+
+ # Test readonly fields from OpenAPI spec
+ readonly_fields = ['uuid', 'date_created', 'last_viewed']
+ for field in readonly_fields:
+ watch.reset_watch_edited_flag()
+ if field in watch:
+ old_value = watch[field]
+ watch[field] = 'modified-readonly-value'
+ assert not watch.was_edited, f"Flag should stay False when modifying readonly field '{field}'"
+ watch[field] = old_value # Restore
+
+ # Test additional system fields not in OpenAPI spec yet
+ system_fields = ['last_check_status']
+ for field in system_fields:
+ watch.reset_watch_edited_flag()
+ watch[field] = 'system-value'
+ assert not watch.was_edited, f"Flag should stay False when modifying system field '{field}'"
+
+ # Test that content-type (readonly per OpenAPI) doesn't trigger flag
+ watch.reset_watch_edited_flag()
+ watch['content-type'] = 'text/html'
+ assert not watch.was_edited, "Flag should stay False when modifying 'content-type' (readonly)"
+
+ print("✓ System fields correctly don't trigger the flag")
diff --git a/changedetectionio/worker.py b/changedetectionio/worker.py
index d253317ff..406890f27 100644
--- a/changedetectionio/worker.py
+++ b/changedetectionio/worker.py
@@ -276,6 +276,9 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
# Yes fine, so nothing todo, don't continue to process.
process_changedetection_results = False
changed_detected = False
+ logger.debug(f'[{uuid}] - checksumFromPreviousCheckWasTheSame - Checksum from previous check was the same, nothing todo here.')
+ # Reset the edited flag since we successfully completed the check
+ watch.reset_watch_edited_flag()
except content_fetchers_exceptions.BrowserConnectError as e:
datastore.update_watch(uuid=uuid,
@@ -392,6 +395,8 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
logger.debug(f"Processing watch UUID: {uuid} - xpath_data length returned {len(update_handler.xpath_data) if update_handler and update_handler.xpath_data else 'empty.'}")
if update_handler and process_changedetection_results:
try:
+ # Reset the edited flag BEFORE update_watch (which calls watch.update() and would set it again)
+ watch.reset_watch_edited_flag()
datastore.update_watch(uuid=uuid, update_obj=update_obj)
if changed_detected or not watch.history_n:
@@ -439,8 +444,22 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
logger.exception(f"Worker {worker_id} full exception details:")
datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
+
# Always record attempt count
count = watch.get('check_count', 0) + 1
+
+ final_updates = {'fetch_time': round(time.time() - fetch_start_time, 3),
+ 'check_count': count,
+ }
+ # Record server header
+ try:
+ server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]
+ if server_header:
+ final_updates['remote_server_reply'] = server_header
+ except Exception as e:
+ server_header = None
+ pass
+
if update_handler: # Could be none or empty if the processor was not found
# Always record page title (used in notifications, and can change even when the content is the same)
if update_obj.get('content-type') and 'html' in update_obj.get('content-type'):
@@ -449,17 +468,12 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
if page_title:
page_title = page_title.strip()[:2000]
logger.debug(f"UUID: {uuid} Page is '{page_title}'")
- datastore.update_watch(uuid=uuid, update_obj={'page_title': page_title})
+ final_updates['page_title'] = page_title
except Exception as e:
logger.exception(f"Worker {worker_id} full exception details:")
logger.warning(f"UUID: {uuid} Exception when extracting - {str(e)}")
- # Record server header
- try:
- server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]
- datastore.update_watch(uuid=uuid, update_obj={'remote_server_reply': server_header})
- except Exception as e:
- pass
+
# Store favicon if necessary
if update_handler.fetcher.favicon_blob and update_handler.fetcher.favicon_blob.get('base64'):
@@ -467,14 +481,12 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
favicon_base_64=update_handler.fetcher.favicon_blob.get('base64')
)
- datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - fetch_start_time, 3),
- 'check_count': count})
+ datastore.update_watch(uuid=uuid, update_obj=final_updates)
# NOW clear fetcher content - after all processing is complete
# This is the last point where we need the fetcher data
if update_handler and hasattr(update_handler, 'fetcher') and update_handler.fetcher:
update_handler.fetcher.clear_content()
- logger.debug(f"Cleared fetcher content for UUID {uuid}")
# Explicitly delete update_handler to free all references
if update_handler: