Merge branch 'master' into datastore-refactor

changedetectionio/__init__.py

@@ -133,48 +133,43 @@ def sigshutdown_handler(_signo, _stack_frame):
     
     sys.exit()
 
-def print_help():
-    """Print help text for command line options"""
-    print('Usage: changedetection.py [options]')
-    print('')
-    print('Standard options:')
-    print('  -s                SSL enable')
-    print('  -h HOST           Listen host (default: 0.0.0.0)')
-    print('  -p PORT           Listen port (default: 5000)')
-    print('  -d PATH           Datastore path')
-    print('  -l LEVEL          Log level (TRACE, DEBUG, INFO, SUCCESS, WARNING, ERROR, CRITICAL)')
-    print('  -c                Cleanup unused snapshots')
-    print('  -C                Create datastore directory if it doesn\'t exist')
-    print('  -P true/false     Set all watches paused (true) or active (false)')
-    print('')
-    print('Add URLs on startup:')
-    print('  -u URL            Add URL to watch (can be used multiple times)')
-    print('  -u0 \'JSON\'        Set options for first -u URL (e.g. \'{"processor":"text_json_diff"}\')')
-    print('  -u1 \'JSON\'        Set options for second -u URL (0-indexed)')
-    print('  -u2 \'JSON\'        Set options for third -u URL, etc.')
-    print('                    Available options: processor, fetch_backend, headers, method, etc.')
-    print('                    See model/Watch.py for all available options')
-    print('')
-    print('Recheck on startup:')
-    print('  -r all            Queue all watches for recheck on startup')
-    print('  -r UUID,...       Queue specific watches (comma-separated UUIDs)')
-    print('  -r all N          Queue all watches, wait for completion, repeat N times')
-    print('  -r UUID,... N     Queue specific watches, wait for completion, repeat N times')
-    print('')
-    print('Batch mode:')
-    print('  -b                Run in batch mode (process queue then exit)')
-    print('                    Useful for CI/CD, cron jobs, or one-time checks')
-    print('                    NOTE: Batch mode checks if Flask is running and aborts if port is in use')
-    print('                    Use -p PORT to specify a different port if needed')
-    print('')
-
 def main():
     global datastore
     global app
 
     # Early help/version check before any initialization
     if '--help' in sys.argv or '-help' in sys.argv:
-        print_help()
+        print('Usage: changedetection.py [options]')
+        print('')
+        print('Standard options:')
+        print('  -s                SSL enable')
+        print('  -h HOST           Listen host (default: 0.0.0.0)')
+        print('  -p PORT           Listen port (default: 5000)')
+        print('  -d PATH           Datastore path')
+        print('  -l LEVEL          Log level (TRACE, DEBUG, INFO, SUCCESS, WARNING, ERROR, CRITICAL)')
+        print('  -c                Cleanup unused snapshots')
+        print('  -C                Create datastore directory if it doesn\'t exist')
+        print('')
+        print('Add URLs on startup:')
+        print('  -u URL            Add URL to watch (can be used multiple times)')
+        print('  -u0 \'JSON\'        Set options for first -u URL (e.g. \'{"processor":"text_json_diff"}\')')
+        print('  -u1 \'JSON\'        Set options for second -u URL (0-indexed)')
+        print('  -u2 \'JSON\'        Set options for third -u URL, etc.')
+        print('                    Available options: processor, fetch_backend, headers, method, etc.')
+        print('                    See model/Watch.py for all available options')
+        print('')
+        print('Recheck on startup:')
+        print('  -r all            Queue all watches for recheck on startup')
+        print('  -r UUID,...       Queue specific watches (comma-separated UUIDs)')
+        print('  -r all N          Queue all watches, wait for completion, repeat N times')
+        print('  -r UUID,... N     Queue specific watches, wait for completion, repeat N times')
+        print('')
+        print('Batch mode:')
+        print('  -b                Run in batch mode (process queue then exit)')
+        print('                    Useful for CI/CD, cron jobs, or one-time checks')
+        print('                    NOTE: Batch mode checks if Flask is running and aborts if port is in use')
+        print('                    Use -p PORT to specify a different port if needed')
+        print('')
         sys.exit(0)
 
     if '--version' in sys.argv or '-v' in sys.argv:
@@ -190,7 +185,6 @@ def main():
     # Set a default logger level
     logger_level = 'DEBUG'
     include_default_watches = True
-    all_paused = None  # None means don't change, True/False to set
 
     host = os.environ.get("LISTEN_HOST", "0.0.0.0").strip()
     port = int(os.environ.get('PORT', 5000))
@@ -269,9 +263,39 @@ def main():
         i += 1
 
     try:
-        opts, args = getopt.getopt(cleaned_argv[1:], "6Ccsd:h:p:l:P:", "port")
+        opts, args = getopt.getopt(cleaned_argv[1:], "6Ccsd:h:p:l:", "port")
     except getopt.GetoptError as e:
-        print_help()
+        print('Usage: changedetection.py [options]')
+        print('')
+        print('Standard options:')
+        print('  -s                SSL enable')
+        print('  -h HOST           Listen host (default: 0.0.0.0)')
+        print('  -p PORT           Listen port (default: 5000)')
+        print('  -d PATH           Datastore path')
+        print('  -l LEVEL          Log level (TRACE, DEBUG, INFO, SUCCESS, WARNING, ERROR, CRITICAL)')
+        print('  -c                Cleanup unused snapshots')
+        print('  -C                Create datastore directory if it doesn\'t exist')
+        print('')
+        print('Add URLs on startup:')
+        print('  -u URL            Add URL to watch (can be used multiple times)')
+        print('  -u0 \'JSON\'        Set options for first -u URL (e.g. \'{"processor":"text_json_diff"}\')')
+        print('  -u1 \'JSON\'        Set options for second -u URL (0-indexed)')
+        print('  -u2 \'JSON\'        Set options for third -u URL, etc.')
+        print('                    Available options: processor, fetch_backend, headers, method, etc.')
+        print('                    See model/Watch.py for all available options')
+        print('')
+        print('Recheck on startup:')
+        print('  -r all            Queue all watches for recheck on startup')
+        print('  -r UUID,...       Queue specific watches (comma-separated UUIDs)')
+        print('  -r all N          Queue all watches, wait for completion, repeat N times')
+        print('  -r UUID,... N     Queue specific watches, wait for completion, repeat N times')
+        print('')
+        print('Batch mode:')
+        print('  -b                Run in batch mode (process queue then exit)')
+        print('                    Useful for CI/CD, cron jobs, or one-time checks')
+        print('                    NOTE: Batch mode checks if Flask is running and aborts if port is in use')
+        print('                    Use -p PORT to specify a different port if needed')
+        print('')
         print(f'Error: {e}')
         sys.exit(2)
 
@@ -308,14 +332,6 @@ def main():
         if opt == '-l':
             logger_level = int(arg) if arg.isdigit() else arg.upper()
 
-        if opt == '-P':
-            try:
-                all_paused = bool(strtobool(arg))
-            except ValueError:
-                print(f'Error: Invalid value for -P option: {arg}')
-                print('Expected: true, false, yes, no, 1, or 0')
-                sys.exit(2)
-
     # If URLs are provided, don't include default watches
     if urls_to_add:
         include_default_watches = False
@@ -382,11 +398,6 @@ def main():
         logger.critical(str(e))
         return
 
-    # Apply all_paused setting if specified via CLI
-    if all_paused is not None:
-        datastore.data['settings']['application']['all_paused'] = all_paused
-        logger.info(f"Setting all watches paused: {all_paused}")
-
     # Inject datastore into plugins that need access to settings
     from changedetectionio.pluggy_interface import inject_datastore_into_plugins
     inject_datastore_into_plugins(datastore)
@@ -642,14 +653,7 @@ def main():
     if os.getenv('USE_X_SETTINGS'):
         logger.info("USE_X_SETTINGS is ENABLED")
         from werkzeug.middleware.proxy_fix import ProxyFix
-        app.wsgi_app = ProxyFix(
-            app.wsgi_app,
-            x_for=1,      # X-Forwarded-For (client IP)
-            x_proto=1,    # X-Forwarded-Proto (http/https)
-            x_host=1,     # X-Forwarded-Host (original host)
-            x_port=1,     # X-Forwarded-Port (original port)
-            x_prefix=1    # X-Forwarded-Prefix (URL prefix)
-        )
+        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
 
 
     # In batch mode, skip starting the HTTP server - just keep workers running

changedetectionio/content_fetchers/playwright.py

@@ -8,9 +8,7 @@ from loguru import logger
 from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
     SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_MAX_TOTAL_HEIGHT, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, FAVICON_FETCHER_JS
 from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
-from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable, \
-    BrowserStepsStepException
-
+from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable
 
 async def capture_full_page_async(page, screenshot_format='JPEG', watch_uuid=None, lock_viewport_elements=False):
     import os
@@ -367,16 +365,7 @@ class fetcher(Fetcher):
             try:
                 # Run Browser Steps here
                 if self.browser_steps_get_valid_steps():
-                    try:
-                        await self.iterate_browser_steps(start_url=url)
-                    except BrowserStepsStepException:
-                        try:
-                            await context.close()
-                            await browser.close()
-                        except Exception as e:
-                            # Fine, could be messy situation
-                            pass
-                        raise
+                    await self.iterate_browser_steps(start_url=url)
 
                     await self.page.wait_for_timeout(extra_wait * 1000)
 
@@ -418,11 +407,19 @@ class fetcher(Fetcher):
                 # Force aggressive memory cleanup - screenshots are large and base64 decode creates temporary buffers
                 await self.page.request_gc()
                 gc.collect()
+                # Release C-level memory from base64 decode back to OS
+                try:
+                    import ctypes
+                    ctypes.CDLL('libc.so.6').malloc_trim(0)
+                except Exception:
+                    pass
 
             except ScreenshotUnavailable:
                 # Re-raise screenshot unavailable exceptions
+                raise
+            except Exception as e:
+                # It's likely the screenshot was too long/big and something crashed
                 raise ScreenshotUnavailable(url=url, status_code=self.status_code)
-
             finally:
                 # Request garbage collection one more time before closing
                 try:

changedetectionio/store/__init__.py

@@ -218,16 +218,21 @@ class ChangeDetectionStore(DatastoreUpdatesMixin, FileSavingDataStore):
                 # Load the legacy datastore to get its schema_version
                 from .legacy_loader import load_legacy_format
                 legacy_path = os.path.join(self.datastore_path, "url-watches.json")
-                with open(legacy_path) as f:
-                    self.__data = json.load(f)
+                legacy_data = load_legacy_format(legacy_path)
 
-                if not self.__data:
+                if not legacy_data:
                     raise Exception("Failed to load legacy datastore from url-watches.json")
 
+                # Get the schema version from legacy datastore (defaults to 0 if not present)
+                legacy_schema_version = legacy_data.get('settings', {}).get('application', {}).get('schema_version', 0)
+                logger.info(f"Legacy datastore schema version: {legacy_schema_version}")
+
+                # Set our schema version to match the legacy one
+                self.__data['settings']['application']['schema_version'] = legacy_schema_version
+
                 # update_26 will load the legacy data again and migrate to new format
                 # Only run updates AFTER the legacy schema version (e.g., if legacy is at 25, only run 26+)
-                self.run_updates()
-
+                self.run_updates(current_schema_version=legacy_schema_version)
 
             else:
                 # Fresh install - create new datastore
@@ -302,7 +307,7 @@ class ChangeDetectionStore(DatastoreUpdatesMixin, FileSavingDataStore):
         else:
             watch_class = get_custom_watch_obj_for_processor(entity.get('processor'))
 
-        if entity.get('processor') != 'text_json_diff':
+        if entity.get('uuid') != 'text_json_diff':
             logger.trace(f"Loading Watch object '{watch_class.__module__}.{watch_class.__name__}' for UUID {uuid}")
 
         entity = watch_class(datastore_path=self.datastore_path, default=entity)
@@ -368,13 +373,6 @@ class ChangeDetectionStore(DatastoreUpdatesMixin, FileSavingDataStore):
         self.__data['watching'] = watching
         self._watch_hashes = watch_hashes
 
-        # Verify all watches have hashes
-        missing_hashes = [uuid for uuid in watching.keys() if uuid not in watch_hashes]
-        if missing_hashes:
-            logger.error(f"WARNING: {len(missing_hashes)} watches missing hashes after load: {missing_hashes[:5]}")
-        else:
-            logger.debug(f"All {len(watching)} watches have valid hashes")
-
     def _delete_watch(self, uuid):
         """
         Delete a watch from storage.

changedetectionio/store/file_saving_datastore.py

@@ -322,9 +322,8 @@ def load_all_watches(datastore_path, rehydrate_entity_func, compute_hash_func):
         watch, raw_data = load_watch_from_file(watch_json, uuid_dir, rehydrate_entity_func)
         if watch and raw_data:
             watching[uuid_dir] = watch
-            # Compute hash from rehydrated Watch object (as dict) to match how we compute on save
-            # This ensures hash matches what audit will compute from dict(watch)
-            watch_hashes[uuid_dir] = compute_hash_func(dict(watch))
+            # Compute hash from raw data BEFORE rehydration to match saved hash
+            watch_hashes[uuid_dir] = compute_hash_func(raw_data)
             loaded += 1
 
             if loaded % 100 == 0:
@@ -744,7 +743,7 @@ class FileSavingDataStore(DataStore):
                             self._dirty_watches.add(uuid)
                             changes_found += 1
                             logger.warning(
-                                f"Audit detected unmarked change in watch {uuid[:8]}... current {current_hash:8} stored hash {stored_hash[:8]}"
+                                f"Audit detected unmarked change in watch {uuid[:8]}... "
                                 f"(hash changed but not marked dirty)"
                             )
                             self.needs_write = True

changedetectionio/store/updates.py

@@ -534,7 +534,7 @@ class DatastoreUpdatesMixin:
                     logger.debug(f"Renaming history index {old_history_txt} to {new_history_txt}...")
                     shutil.move(old_history_txt, new_history_txt)
 
-    def migrate_legacy_db_format(self):
+    def update_26(self):
         """
         Migration: Individual watch persistence (COPY-based, safe rollback).
 
@@ -578,6 +578,25 @@ class DatastoreUpdatesMixin:
 
         # Populate settings from legacy data
         logger.info("Populating settings from legacy data...")
+        if 'settings' in legacy_data:
+            self.data['settings'] = legacy_data['settings']
+        if 'app_guid' in legacy_data:
+            self.data['app_guid'] = legacy_data['app_guid']
+        if 'build_sha' in legacy_data:
+            self.data['build_sha'] = legacy_data['build_sha']
+        if 'version_tag' in legacy_data:
+            self.data['version_tag'] = legacy_data['version_tag']
+
+        # Rehydrate watches from legacy data
+        logger.info("Rehydrating watches from legacy data...")
+        self.data['watching'] = {}
+        for uuid, watch_data in legacy_data.get('watching', {}).items():
+            try:
+                self.data['watching'][uuid] = self.rehydrate_entity(uuid, watch_data)
+            except Exception as e:
+                logger.error(f"Failed to rehydrate watch {uuid}: {e}")
+                raise Exception(f"Migration failed: Could not rehydrate watch {uuid}. Error: {e}")
+
         watch_count = len(self.data['watching'])
         logger.success(f"Loaded {watch_count} watches from legacy format")
 
@@ -590,10 +609,12 @@ class DatastoreUpdatesMixin:
                 watch_dict = dict(watch)
                 watch_dir = os.path.join(self.datastore_path, uuid)
                 save_watch_atomic(watch_dir, uuid, watch_dict)
+                # Initialize hash
+                self._watch_hashes[uuid] = self._compute_hash(watch_dict)
                 saved_count += 1
 
                 if saved_count % 100 == 0:
-                    logger.info(f"  Progress: {saved_count}/{watch_count} watches migrated...")
+                    logger.info(f"  Progress: {saved_count}/{watch_count} watches saved...")
 
             except Exception as e:
                 logger.error(f"Failed to save watch {uuid}: {e}")
@@ -646,25 +667,9 @@ class DatastoreUpdatesMixin:
 
         # Success! Now reload from new format
         logger.critical("Reloading datastore from new format...")
-        self._load_state() # Includes load_watches
+        self._load_state()
         logger.success("Datastore reloaded from new format successfully")
 
-
-        # Verify all watches have hashes after migration
-        missing_hashes = [uuid for uuid in self.data['watching'].keys() if uuid not in self._watch_hashes]
-        if missing_hashes:
-            logger.error(f"WARNING: {len(missing_hashes)} watches missing hashes after migration: {missing_hashes[:5]}")
-        else:
-            logger.success(f"All {len(self.data['watching'])} watches have valid hashes after migration")
-
-        # Set schema version to latest available update
-        # This prevents re-running updates and re-marking all watches as dirty
-        updates_available = self.get_updates_available()
-        latest_schema = updates_available[-1] if updates_available else 26
-        self.data['settings']['application']['schema_version'] = latest_schema
-        self.mark_settings_dirty()
-        logger.info(f"Set schema_version to {latest_schema} (migration complete, all watches already saved)")
-
         logger.critical("=" * 80)
         logger.critical("MIGRATION COMPLETED SUCCESSFULLY!")
         logger.critical("=" * 80)
@@ -682,5 +687,4 @@ class DatastoreUpdatesMixin:
         logger.info(f"  - rm {os.path.join(self.datastore_path, 'url-watches.json')}")
         logger.info("")
 
-    def update_26(self):
-        self.migrate_legacy_db_format()
+        # Schema version will be updated by run_updates()

changedetectionio/test_cli_opts.sh

@@ -70,8 +70,8 @@ test_single_url() {
     local test_id=$1
     local dir="/tmp/cli-test-single-${test_id}-$$"
     timeout 10 python3 changedetection.py -d "$dir" -C -u https://example.com -b &>/dev/null
-    # Count watch directories (UUID directories containing watch.json)
-    [ "$(find "$dir" -mindepth 2 -maxdepth 2 -name 'watch.json' | wc -l)" -eq 1 ]
+    [ -f "$dir/url-watches.json" ] && \
+    [ "$(python3 -c "import json; print(len(json.load(open('$dir/url-watches.json')).get('watching', {})))")" -eq 1 ]
 }
 
 test_multiple_urls() {
@@ -82,8 +82,8 @@ test_multiple_urls() {
         -u https://github.com \
         -u https://httpbin.org \
         -b &>/dev/null
-    # Count watch directories (UUID directories containing watch.json)
-    [ "$(find "$dir" -mindepth 2 -maxdepth 2 -name 'watch.json' | wc -l)" -eq 3 ]
+    [ -f "$dir/url-watches.json" ] && \
+    [ "$(python3 -c "import json; print(len(json.load(open('$dir/url-watches.json')).get('watching', {})))")" -eq 3 ]
 }
 
 test_url_with_options() {
@@ -93,17 +93,8 @@ test_url_with_options() {
         -u https://example.com \
         -u0 '{"title":"Test Site","processor":"text_json_diff"}' \
         -b &>/dev/null
-    # Check that at least one watch.json contains the title "Test Site"
-    python3 -c "
-import json, glob, sys
-watch_files = glob.glob('$dir/*/watch.json')
-for wf in watch_files:
-    with open(wf) as f:
-        data = json.load(f)
-        if data.get('title') == 'Test Site':
-            sys.exit(0)
-sys.exit(1)
-"
+    [ -f "$dir/url-watches.json" ] && \
+    python3 -c "import json; data=json.load(open('$dir/url-watches.json')); watches=data.get('watching', {}); exit(0 if any(w.get('title')=='Test Site' for w in watches.values()) else 1)"
 }
 
 test_multiple_urls_with_options() {
@@ -115,19 +106,9 @@ test_multiple_urls_with_options() {
         -u https://github.com \
         -u1 '{"title":"Site Two"}' \
         -b &>/dev/null
-    # Check that we have 2 watches and both titles are present
-    python3 -c "
-import json, glob, sys
-watch_files = glob.glob('$dir/*/watch.json')
-if len(watch_files) != 2:
-    sys.exit(1)
-titles = []
-for wf in watch_files:
-    with open(wf) as f:
-        data = json.load(f)
-        titles.append(data.get('title'))
-sys.exit(0 if 'Site One' in titles and 'Site Two' in titles else 1)
-"
+    [ -f "$dir/url-watches.json" ] && \
+    [ "$(python3 -c "import json; print(len(json.load(open('$dir/url-watches.json')).get('watching', {})))")" -eq 2 ] && \
+    python3 -c "import json; data=json.load(open('$dir/url-watches.json')); watches=data.get('watching', {}); titles=[w.get('title') for w in watches.values()]; exit(0 if 'Site One' in titles and 'Site Two' in titles else 1)"
 }
 
 test_batch_mode_exit() {
@@ -145,24 +126,21 @@ test_batch_mode_exit() {
 test_recheck_all() {
     local test_id=$1
     local dir="/tmp/cli-test-recheck-all-${test_id}-$$"
-    # Create a watch using CLI, then recheck it
-    timeout 10 python3 changedetection.py -d "$dir" -C -u https://example.com -b &>/dev/null
-    # Now recheck all watches
-    timeout 10 python3 changedetection.py -d "$dir" -r all -b 2>&1 | grep -q "Queuing"
+    mkdir -p "$dir"
+    cat > "$dir/url-watches.json" << 'EOF'
+{"watching":{"test-uuid":{"url":"https://example.com","last_checked":0,"processor":"text_json_diff","uuid":"test-uuid"}},"settings":{"application":{"password":false}}}
+EOF
+    timeout 10 python3 changedetection.py -d "$dir" -r all -b 2>&1 | grep -q "Queuing all"
 }
 
 test_recheck_specific() {
     local test_id=$1
     local dir="/tmp/cli-test-recheck-uuid-${test_id}-$$"
-    # Create 2 watches using CLI
-    timeout 12 python3 changedetection.py -d "$dir" -C \
-        -u https://example.com \
-        -u https://github.com \
-        -b &>/dev/null
-    # Get the UUIDs that were created
-    local uuids=$(find "$dir" -mindepth 2 -maxdepth 2 -name 'watch.json' -exec dirname {} \; | xargs -n1 basename | tr '\n' ',' | sed 's/,$//')
-    # Now recheck specific UUIDs
-    timeout 10 python3 changedetection.py -d "$dir" -r "$uuids" -b 2>&1 | grep -q "Queuing"
+    mkdir -p "$dir"
+    cat > "$dir/url-watches.json" << 'EOF'
+{"watching":{"uuid-1":{"url":"https://example.com","last_checked":0,"processor":"text_json_diff","uuid":"uuid-1"},"uuid-2":{"url":"https://github.com","last_checked":0,"processor":"text_json_diff","uuid":"uuid-2"}},"settings":{"application":{"password":false}}}
+EOF
+    timeout 10 python3 changedetection.py -d "$dir" -r uuid-1,uuid-2 -b 2>&1 | grep -q "Queuing 2 specific watches"
 }
 
 test_combined_operations() {
@@ -173,8 +151,8 @@ test_combined_operations() {
         -u https://github.com \
         -r all \
         -b &>/dev/null
-    # Count watch directories (UUID directories containing watch.json)
-    [ "$(find "$dir" -mindepth 2 -maxdepth 2 -name 'watch.json' | wc -l)" -eq 2 ]
+    [ -f "$dir/url-watches.json" ] && \
+    [ "$(python3 -c "import json; print(len(json.load(open('$dir/url-watches.json')).get('watching', {})))")" -eq 2 ]
 }
 
 test_invalid_json() {