WIP

changedetectionio/__init__.py

@@ -33,7 +33,6 @@ def sigshutdown_handler(_signo, _stack_frame):
     global datastore
     name = signal.Signals(_signo).name
     logger.critical(f'Shutdown: Got Signal - {name} ({_signo}), Saving DB to disk and calling shutdown')
-    datastore.sync_to_json()
     logger.success('Sync JSON to disk complete.')
     # This will throw a SystemExit exception, because eventlet.wsgi.server doesn't know how to deal with it.
     # Solution: move to gevent or other server in the future (#2014)

changedetectionio/api/api_v1.py

@@ -126,6 +126,7 @@ class Watch(Resource):
                 return f"Invalid proxy choice, currently supported proxies are '{', '.join(plist)}'", 400
 
         self.datastore.data['watching'][uuid].update(request.json)
+        self.datastore.data['watching'][uuid].save_data()
 
         return "OK", 200
 

changedetectionio/blueprint/backups/__init__.py

@@ -89,8 +89,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             flash("Maximum number of backups reached, please remove some", "error")
             return redirect(url_for('backups.index'))
 
-        # Be sure we're written fresh
-        datastore.sync_to_json()
         zip_thread = threading.Thread(target=create_backup, args=(datastore.datastore_path, datastore.data.get("watching")))
         zip_thread.start()
         backup_threads.append(zip_thread)

changedetectionio/blueprint/settings/__init__.py

@@ -71,12 +71,12 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
                 if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password):
                     datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password
-                    datastore.needs_write_urgent = True
+                    datastore.save_settings()
                     flash("Password protection enabled.", 'notice')
                     flask_login.logout_user()
                     return redirect(url_for('index'))
 
-                datastore.needs_write_urgent = True
+                datastore.save_settings()
                 flash("Settings updated.")
 
             else:
@@ -100,8 +100,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
             datastore.data['settings']['application'].update(app_update)
             datastore.data['settings']['requests'].update(form.data['requests'])
-
-            datastore.needs_write_urgent = True
+            datastore.save_settings()
             flash("Settings updated.")
 
         output = render_template("settings.html",
@@ -125,7 +124,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def settings_reset_api_key():
         secret = secrets.token_hex(16)
         datastore.data['settings']['application']['api_access_token'] = secret
-        datastore.needs_write_urgent = True
         flash("API Key was regenerated.")
         return redirect(url_for('settings.settings_page')+'#api')
         

changedetectionio/blueprint/tags/__init__.py

@@ -56,6 +56,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def mute(uuid):
         if datastore.data['settings']['application']['tags'].get(uuid):
             datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = not datastore.data['settings']['application']['tags'][uuid]['notification_muted']
+            datastore.data['settings']['application']['tags'][uuid].save_data()
         return redirect(url_for('tags.tags_overview_page'))
 
     @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
@@ -176,7 +177,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         datastore.data['settings']['application']['tags'][uuid].update(form.data)
         datastore.data['settings']['application']['tags'][uuid]['processor'] = 'restock_diff'
-        datastore.needs_write_urgent = True
+        datastore.data['settings']['application']['tags'][uuid].save_data()
+
         flash("Updated")
 
         return redirect(url_for('tags.tags_overview_page'))

changedetectionio/blueprint/ui/__init__.py

@@ -163,6 +163,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
                 uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
                     datastore.data['watching'][uuid.strip()]['paused'] = True
+                    datastore.data['watching'][uuid.strip()].save_data()
             flash("{} watches paused".format(len(uuids)))
 
         elif (op == 'unpause'):
@@ -170,6 +171,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
                 uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
                     datastore.data['watching'][uuid.strip()]['paused'] = False
+                    datastore.data['watching'][uuid.strip()].save_data()
             flash("{} watches unpaused".format(len(uuids)))
 
         elif (op == 'mark-viewed'):
@@ -184,6 +186,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
                 uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
                     datastore.data['watching'][uuid.strip()]['notification_muted'] = True
+                    datastore.data['watching'][uuid.strip()].save_data()
             flash("{} watches muted".format(len(uuids)))
 
         elif (op == 'unmute'):
@@ -191,6 +194,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
                 uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
                     datastore.data['watching'][uuid.strip()]['notification_muted'] = False
+                    datastore.data['watching'][uuid.strip()].save_data()
             flash("{} watches un-muted".format(len(uuids)))
 
         elif (op == 'recheck'):
@@ -206,6 +210,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
                 uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
                     datastore.data['watching'][uuid]["last_error"] = False
+                    datastore.data['watching'][uuid].save_data()
             flash(f"{len(uuids)} watches errors cleared")
 
         elif (op == 'clear-history'):
@@ -244,6 +249,9 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat
 
             flash(f"{len(uuids)} watches were tagged")
 
+        for uuid in uuids:
+            datastore.data['watching'][uuid.strip()].save_data()
+
         return redirect(url_for('index'))
 
 

changedetectionio/blueprint/ui/edit.py

@@ -49,8 +49,8 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                     datastore.clear_watch_history(uuid)
                     redirect(url_for('ui_edit.edit_page', uuid=uuid))
 
-        # be sure we update with a copy instead of accidently editing the live object by reference
-        default = deepcopy(datastore.data['watching'][uuid])
+
+        default = datastore.data['watching'][uuid]
 
         # Defaults for proxy choice
         if datastore.proxy_list is not None:  # When enabled
@@ -114,10 +114,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                 extra_update_obj['paused'] = False
 
             extra_update_obj['time_between_check'] = form.time_between_check.data
-
-             # Ignore text
-            form_ignore_text = form.ignore_text.data
-            datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text
+            extra_update_obj['ignore_text'] = form.ignore_text.data
 
             # Be sure proxy value is None
             if datastore.proxy_list is not None and form.data['proxy'] == '':
@@ -143,23 +140,23 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                         tag_uuids.append(datastore.add_tag(name=t))
                     extra_update_obj['tags'] = tag_uuids
 
-            datastore.data['watching'][uuid].update(form.data)
-            datastore.data['watching'][uuid].update(extra_update_obj)
+
+
 
             if not datastore.data['watching'][uuid].get('tags'):
                 # Force it to be a list, because form.data['tags'] will be string if nothing found
                 # And del(form.data['tags'] ) wont work either for some reason
                 datastore.data['watching'][uuid]['tags'] = []
 
+            datastore.update_watch(uuid=uuid, update_obj=form.data | extra_update_obj)
+
             # Recast it if need be to right data Watch handler
-            processor_name = form.data.get('processor')
+            processor_name = datastore.data['watching'][uuid].get('processor')
             watch_class = processors.get_watch_model_for_processor(processor_name)
             datastore.data['watching'][uuid] = watch_class(datastore_path=datastore.datastore_path, default=datastore.data['watching'][uuid])
+            datastore.data['watching'][uuid].save_data()
             flash("Updated watch - unpaused!" if request.args.get('unpause_on_save') else "Updated watch.")
 
-            # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
-            # But in the case something is added we should save straight away
-            datastore.needs_write_urgent = True
 
             # Do not queue on edit if its not within the time range
 
@@ -186,6 +183,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                         f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}")
                     return False
 
+
             #############################
             if not datastore.data['watching'][uuid].get('paused') and is_in_schedule:
                 # Queue the watch for immediate recheck, with a higher priority

changedetectionio/flask_app.py

@@ -370,7 +370,6 @@ def changedetection_app(config=None, datastore_o=None):
             elif op == 'mute':
                 datastore.data['watching'][uuid].toggle_mute()
 
-            datastore.needs_write = True
             return redirect(url_for('index', tag = active_tag_uuid))
 
         # Sort by last_changed and add the uuid which is usually the key..

changedetectionio/model/Tag.py

@@ -1,14 +1,57 @@
 
-from changedetectionio.model import watch_base
+import os
+import json
+import uuid as uuid_builder
+import time
+from copy import deepcopy
+from loguru import logger
+
+from changedetectionio.model import watch_base, schema
 
 
 class model(watch_base):
+    """Tag model that writes to tags/{uuid}/tag.json instead of the main watch directory"""
+    __datastore_path = None
 
     def __init__(self, *arg, **kw):
         super(model, self).__init__(*arg, **kw)
+        self.__datastore_path = kw.get("datastore_path")
 
         self['overrides_watch'] = kw.get('default', {}).get('overrides_watch')
 
         if kw.get('default'):
             self.update(kw['default'])
             del kw['default']
+
+    @property
+    def watch_data_dir(self):
+        # Override to use tags directory instead of the normal watch data directory
+        datastore_path = getattr(self, '_model__datastore_path', None)
+        if datastore_path:
+            tags_path = os.path.join(datastore_path, 'tags')
+            # Make sure the tags directory exists
+            if not os.path.exists(tags_path):
+                os.makedirs(tags_path)
+            return os.path.join(tags_path, self['uuid'])
+        return None
+        
+    def save_data(self):
+        """Override to save tag to tags/{uuid}/tag.json"""
+        logger.debug(f"Saving tag {self['uuid']}")
+
+        if not self.get('uuid'):
+            # Might have been called when creating the tag
+            return
+
+        tags_path = os.path.join(self.__datastore_path, 'tags')
+        if not os.path.isdir(tags_path):
+            os.mkdir(os.path.join(tags_path))
+
+        path = os.path.join(tags_path, self.get('uuid')+".json")
+        try:
+            with open(path + ".tmp", 'w') as json_file:
+                json.dump(self.get_data(), json_file, indent=4)
+            os.replace(path + ".tmp", path)
+        except Exception as e:
+            logger.error(f"Error writing JSON for tag {self.get('uuid')}!! (JSON file save was skipped) : {str(e)}")
+

changedetectionio/model/Watch.py

@@ -38,18 +38,13 @@ class model(watch_base):
     jitter_seconds = 0
 
     def __init__(self, *arg, **kw):
-        self.__datastore_path = kw.get('datastore_path')
-        if kw.get('datastore_path'):
-            del kw['datastore_path']
+
         super(model, self).__init__(*arg, **kw)
 
         if kw.get('default'):
             self.update(kw['default'])
             del kw['default']
 
-        if self.get('default'):
-            del self['default']
-
         # Be sure the cached timestamp is ready
         bump = self.history
 
@@ -301,6 +296,7 @@ class model(watch_base):
     # result_obj from fetch_site_status.run()
     def save_history_text(self, contents, timestamp, snapshot_id):
         import brotli
+        import tempfile
 
         logger.trace(f"{self.get('uuid')} - Updating history.txt with timestamp {timestamp}")
 
@@ -417,11 +413,6 @@ class model(watch_base):
     def snapshot_error_screenshot_ctime(self):
         return self.__get_file_ctime('last-error-screenshot.png')
 
-    @property
-    def watch_data_dir(self):
-        # The base dir of the watch data
-        return os.path.join(self.__datastore_path, self['uuid']) if self.__datastore_path else None
-
     def get_error_text(self):
         """Return the text saved from a previous request that resulted in a non-200 error"""
         fname = os.path.join(self.watch_data_dir, "last-error.txt")

changedetectionio/model/__init__.py

@@ -1,7 +1,9 @@
 import os
 import uuid
 from copy import deepcopy
+from loguru import logger
 import time
+import json
 
 from changedetectionio import strtobool
 from changedetectionio.notification import default_notification_format_for_watch
@@ -49,6 +51,7 @@ schema = {
     'previous_md5': False,
     'previous_md5_before_filters': False,  # Used for skipping changedetection entirely
     'processor': 'text_json_diff',  # could be restock_diff or others from .processors
+    'processor_state': {}, # Extra configs for custom processors/plugins, keyed by processor name
     'price_change_threshold_percent': None,
     'proxy': None,  # Preferred proxy connection
     'remote_server_reply': None,  # From 'server' reply header
@@ -131,12 +134,14 @@ schema = {
 
 class watch_base(dict):
     __data = {}
+    __datastore_path = None
+    __save_enabled = True
 
     def __init__(self, *arg, **kw):
         # Initialize internal data storage
 
         self.__data = deepcopy(schema)
-
+        self.__datastore_path = kw.pop('datastore_path', None)
         # Initialize as empty dict but maintain dict interface
         super(watch_base, self).__init__()
         
@@ -147,7 +152,18 @@ class watch_base(dict):
         # Generate UUID if needed
         if not self.__data.get('uuid'):
             self.__data['uuid'] = str(uuid.uuid4())
-    
+
+        if self.__data.get('default'):
+            del(self.__data['default'])
+
+    @property
+    def watch_data_dir(self):
+        # The base dir of the watch data
+        return os.path.join(self.__datastore_path, self['uuid']) if self.__datastore_path else None
+
+    def enable_saving(self):
+        self.__save_enabled = True
+
     # Dictionary interface methods to use self.__data
     def __getitem__(self, key):
         return self.__data[key]
@@ -155,7 +171,7 @@ class watch_base(dict):
     def __setitem__(self, key, value):
         self.__data[key] = value
         self.__data['last_modified'] = time.time()
-    
+
     def __delitem__(self, key):
         del self.__data[key]
     
@@ -205,3 +221,26 @@ class watch_base(dict):
     def get_data(self):
         """Returns the internal data dictionary"""
         return self.__data
+
+    def save_data(self):
+        if self.__save_enabled:
+            if not self.__data.get('uuid'):
+                # Might have been called when creating the watch
+                return
+
+            logger.debug(f"Saving watch {self['uuid']}")
+            path = os.path.join(self.__datastore_path, self.get('uuid'))
+            filepath = os.path.join(str(path), "watch.json")
+            if not os.path.exists(path):
+                os.mkdir(path)
+
+            try:
+                import tempfile
+                with tempfile.NamedTemporaryFile(mode='wb+', delete=False) as tmp:
+                    tmp.write(json.dumps(self.get_data(), indent=2).encode('utf-8'))
+                    tmp.flush()
+                    os.replace(tmp.name, filepath)
+
+
+            except Exception as e:
+                logger.error(f"Error writing JSON for {self.get('uuid')}!! (JSON file save was skipped) : {str(e)}")

changedetectionio/processors/restock_diff/__init__.py

@@ -56,8 +56,13 @@ class Restock(dict):
         super().__setitem__(key, value)
 
 class Watch(BaseWatch):
+
+    def load_extra_vars(self):
+        # something from disk?
+
     def __init__(self, *arg, **kw):
         super().__init__(*arg, **kw)
+        # Restock Obj helps with the state of the situation
         self['restock'] = Restock(kw['default']['restock']) if kw.get('default') and kw['default'].get('restock') else Restock()
 
         self['restock_settings'] = kw['default']['restock_settings'] if kw.get('default',{}).get('restock_settings') else {

changedetectionio/store.py

@@ -31,11 +31,6 @@ dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
 # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
 class ChangeDetectionStore:
     lock = Lock()
-    # For general updates/writes that can wait a few seconds
-    needs_write = False
-
-    # For when we edit, we should write to disk
-    needs_write_urgent = False
 
     __version_check = True
 
@@ -46,7 +41,7 @@ class ChangeDetectionStore:
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
         logger.info(f"Datastore path is '{self.json_store_path}'")
-        self.needs_write = False
+
         self.start_time = time.time()
         self.stop_thread = False
 
@@ -56,39 +51,30 @@ class ChangeDetectionStore:
                 # So when someone gives us a backup file to examine, we know exactly what code they were running.
                 self.__data['build_sha'] = f.read()
 
+        self.generic_definition = deepcopy(Watch.model(datastore_path = datastore_path, default={}))
+
         try:
-            # @todo retest with ", encoding='utf-8'"
-            with open(self.json_store_path) as json_file:
-                from_disk = json.load(json_file)
+            import os
+            # First load global settings from the main JSON file if it exists
+            if os.path.isfile(self.json_store_path):
+                with open(self.json_store_path) as json_file:
+                    from_disk = json.load(json_file)
+                    
+                    # Load app_guid and settings from the main JSON file
+                    if 'app_guid' in from_disk:
+                        self.__data['app_guid'] = from_disk['app_guid']
+    
+                    if 'settings' in from_disk:
+                        if 'headers' in from_disk['settings']:
+                            self.__data['settings']['headers'].update(from_disk['settings']['headers'])
+    
+                        if 'requests' in from_disk['settings']:
+                            self.__data['settings']['requests'].update(from_disk['settings']['requests'])
+    
+                        if 'application' in from_disk['settings']:
+                            self.__data['settings']['application'].update(from_disk['settings']['application'])
 
-                # @todo isnt there a way todo this dict.update recursively?
-                # Problem here is if the one on the disk is missing a sub-struct, it wont be present anymore.
-                if 'watching' in from_disk:
-                    self.__data['watching'].update(from_disk['watching'])
 
-                if 'app_guid' in from_disk:
-                    self.__data['app_guid'] = from_disk['app_guid']
-
-                if 'settings' in from_disk:
-                    if 'headers' in from_disk['settings']:
-                        self.__data['settings']['headers'].update(from_disk['settings']['headers'])
-
-                    if 'requests' in from_disk['settings']:
-                        self.__data['settings']['requests'].update(from_disk['settings']['requests'])
-
-                    if 'application' in from_disk['settings']:
-                        self.__data['settings']['application'].update(from_disk['settings']['application'])
-
-                # Convert each existing watch back to the Watch.model object
-                for uuid, watch in self.__data['watching'].items():
-                    self.__data['watching'][uuid] = self.rehydrate_entity(default_dict=watch)
-                    logger.info(f"Watching: {uuid} {watch['url']}")
-
-                # And for Tags also, should be Restock type because it has extra settings
-                # @todo make this smarter!
-                for uuid, tag in self.__data['settings']['application']['tags'].items():
-                    self.__data['settings']['application']['tags'][uuid] = self.rehydrate_entity(default_dict=tag, processor_override='restock_diff')
-                    logger.info(f"Tag: {uuid} {tag['title']}")
 
         # First time ran, Create the datastore.
         except (FileNotFoundError):
@@ -107,6 +93,8 @@ class ChangeDetectionStore:
 
         else:
             # Bump the update version by running updates
+            self.scan_load_watches()
+            self.scan_load_tags()
             self.run_updates()
 
         self.__data['version_tag'] = version_tag
@@ -138,10 +126,53 @@ class ChangeDetectionStore:
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['api_access_token'] = secret
 
-        self.needs_write = True
+    def scan_load_watches(self):
 
-        # Finally start the thread that will manage periodic data saves to JSON
-        save_data_thread = threading.Thread(target=self.save_datastore).start()
+        # Now scan for individual watch.json files in the datastore directory
+        import pathlib
+        watch_jsons = list(pathlib.Path(self.datastore_path).rglob("*/watch.json"))
+
+        for watch_file in watch_jsons:
+            # Extract UUID from the directory name (parent directory of watch.json)
+            uuid = watch_file.parent.name
+
+            try:
+                with open(watch_file, 'r') as f:
+                    watch_data = json.load(f)
+                    # Create a Watch object and add it to the datastore
+                    self.__data['watching'][uuid] = self.rehydrate_entity(default_dict=watch_data)
+                    logger.info(f"Watching: {uuid} {watch_data.get('url')}")
+
+            except Exception as e:
+                logger.error(f"Error loading watch from {watch_file}: {str(e)}")
+                continue
+        logger.debug(f"{len(self.__data['watching'])} watches loaded.")
+
+    def scan_load_tags(self):
+        import pathlib
+        # Now scan for individual tag.json files in the tags directory
+        tags_path = os.path.join(self.datastore_path, 'tags')
+        if os.path.exists(tags_path):
+            tag_jsons = list(pathlib.Path(tags_path).rglob("*.json"))
+
+            for tag_file in tag_jsons:
+                # Extract UUID from the directory name (parent directory of tag.json)
+
+                try:
+                    with open(tag_file, 'r') as f:
+                        tag_data = json.load(f)
+                        uuid = str(tag_file).replace('.json', '')
+                        tag_data['uuid'] = uuid
+                        # Create a Tag object and add it to the datastore
+                        self.__data['settings']['application']['tags'][uuid] = self.rehydrate_entity(
+                            default_dict=tag_data,
+                            processor_override='restock_diff'
+                        )
+                        logger.info(f"Tag: {uuid} {tag_data.get('title', 'No title found')}")
+                except Exception as e:
+                    logger.error(f"Error loading tag from {tag_file}: {str(e)}")
+                    continue
+        logger.debug(f"{len(self.__data['settings']['application']['tags'])} tags loaded.")
 
     def rehydrate_entity(self, default_dict: dict, processor_override=None):
 
@@ -152,16 +183,17 @@ class ChangeDetectionStore:
         watch_class = get_watch_model_for_processor(processor_override)
         default_dict['processor'] = processor_override
         entity = watch_class(datastore_path=self.datastore_path, default=default_dict)
+        entity.enable_saving()
         return entity
 
     def set_last_viewed(self, uuid, timestamp):
         logger.debug(f"Setting watch UUID: {uuid} last viewed to {int(timestamp)}")
         self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
-        self.needs_write = True
+        self.data['watching'][uuid].save_data()
 
     def remove_password(self):
         self.__data['settings']['application']['password'] = False
-        self.needs_write = True
+        self.save_settings()
 
     def update_watch(self, uuid, update_obj):
         """
@@ -171,21 +203,16 @@ class ChangeDetectionStore:
         if not uuid in self.data['watching'].keys() or update_obj is None:
             return
 
-        with self.lock:
-            # Make sure we're working with a proper Watch object
-            watch = self.data['watching'].get(uuid)
-            
-            # Handle None values - they mean "delete this key"
-            keys_to_remove = [k for k, v in update_obj.items() if v is None]
-            for k in keys_to_remove:
-                if k in watch:
-                    del watch[k]
-                del update_obj[k]
-            
-            # Deep merge with the rest
-            always_merger.merge(watch, update_obj)
-            
-        self.needs_write = True
+        # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures...
+        for dict_key, d in self.generic_definition.items():
+            if isinstance(d, dict):
+                if update_obj is not None and dict_key in update_obj:
+                    self.__data['watching'][uuid][dict_key].update(update_obj[dict_key])
+                    del (update_obj[dict_key])
+
+        self.__data['watching'][uuid].update(update_obj)
+        self.__data['watching'][uuid].save_data()
+
 
     @property
     def threshold_seconds(self):
@@ -245,8 +272,6 @@ class ChangeDetectionStore:
                     shutil.rmtree(path)
                 del self.data['watching'][uuid]
 
-        self.needs_write_urgent = True
-
     # Clone a watch by UUID
     def clone(self, uuid):
         url = self.data['watching'][uuid].get('url')
@@ -266,7 +291,6 @@ class ChangeDetectionStore:
     # Remove a watchs data but keep the entry (URL etc)
     def clear_watch_history(self, uuid):
         self.__data['watching'][uuid].clear_watch()
-        self.needs_write_urgent = True
 
     def add_watch(self, url, tag='', extras=None, tag_uuids=None, write_to_disk_now=True):
         import requests
@@ -357,15 +381,11 @@ class ChangeDetectionStore:
 
         if not apply_extras.get('date_created'):
             apply_extras['date_created'] = int(time.time())
-
-        new_watch.update(apply_extras)
         new_watch.ensure_data_dir_exists()
+        new_watch.update(apply_extras)
+
         self.__data['watching'][new_uuid] = new_watch
-
-
-        if write_to_disk_now:
-            self.sync_to_json()
-
+        self.__data['watching'][new_uuid].save_data()
         logger.debug(f"Added '{url}'")
 
         return new_uuid
@@ -379,58 +399,22 @@ class ChangeDetectionStore:
 
         return False
 
-    def sync_to_json(self):
-        logger.info("Saving JSON..")
+    def save_settings(self):
+        logger.info("Saving application settings...")
         try:
-            data = deepcopy(self.__data)
-        except RuntimeError as e:
-            # Try again in 15 seconds
-            time.sleep(15)
-            logger.error(f"! Data changed when writing to JSON, trying again.. {str(e)}")
-            self.sync_to_json()
-            return
-        else:
-
-            try:
-                # Re #286  - First write to a temp file, then confirm it looks OK and rename it
-                # This is a fairly basic strategy to deal with the case that the file is corrupted,
-                # system was out of memory, out of RAM etc
-                with open(self.json_store_path+".tmp", 'w') as json_file:
-                    json.dump(data, json_file, indent=4)
-                os.replace(self.json_store_path+".tmp", self.json_store_path)
-            except Exception as e:
-                logger.error(f"Error writing JSON!! (Main JSON file save was skipped) : {str(e)}")
-
-            self.needs_write = False
-            self.needs_write_urgent = False
-
-    # Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON
-    # by just running periodically in one thread, according to python, dict updates are threadsafe.
-    def save_datastore(self):
-
-        while True:
-            if self.stop_thread:
-                # Suppressing "Logging error in Loguru Handler #0" during CICD.
-                # Not a meaningful difference for a real use-case just for CICD.
-                # the side effect is a "Shutting down datastore thread" message
-                # at the end of each test.
-                # But still more looking better.
-                import sys
-                logger.remove()
-                logger.add(sys.stderr)
-
-                logger.info("Shutting down datastore thread")
-                return
-
-            if self.needs_write or self.needs_write_urgent:
-                self.sync_to_json()
-
-            # Once per minute is enough, more and it can cause high CPU usage
-            # better here is to use something like self.app.config.exit.wait(1), but we cant get to 'app' from here
-            for i in range(120):
-                time.sleep(0.5)
-                if self.stop_thread or self.needs_write_urgent:
-                    break
+            # Only save app settings, not the watches or tags (they're saved individually)
+            data = {'settings': self.__data.get('settings')}
+            #data = deepcopy(self.__data)
+            
+            # Remove the watches from the main JSON file
+            if 'watching' in data:
+                del data['watching']
+                
+            # Remove the tags from the main JSON file since they're saved individually now
+#            if 'settings' in data and 'application' in data['settings'] and 'tags' in data['settings']['application']:
+#                del data['settings']['application']['tags']
+        except Exception as e:
+            x=1
 
     # Go through the datastore path and remove any snapshots that are not mentioned in the index
     # This usually is not used, but can be handy.
@@ -584,16 +568,17 @@ class ChangeDetectionStore:
 
         # Eventually almost everything todo with a watch will apply as a Tag
         # So we use the same model as a Watch
-        with self.lock:
-            from .model import Tag
-            new_tag = Tag.model(datastore_path=self.datastore_path, default={
-                'title': name.strip(),
-                'date_created': int(time.time())
-            })
+        from .model import Tag
+        new_tag = Tag.model(datastore_path=self.datastore_path, default={
+            'title': name.strip(),
+            'date_created': int(time.time())
+        })
 
-            new_uuid = new_tag.get('uuid')
+        new_uuid = new_tag.get('uuid')
+
+        self.__data['settings']['application']['tags'][new_uuid] = new_tag
+        self.__data['settings']['application']['tags'][new_uuid].save_data()
 
-            self.__data['settings']['application']['tags'][new_uuid] = new_tag
 
         return new_uuid
 

changedetectionio/tests/test_conditions.py

@@ -50,7 +50,7 @@ def test_conditions_with_text_and_number(client, live_server):
     """Test that both text and number conditions work together with AND logic."""
     
     set_original_response("50")
-#    live_server_setup(live_server)
+    #live_server_setup(live_server)
 
     test_url = url_for('test_endpoint', _external=True)
 

changedetectionio/tests/test_restock_save_load_settings.py

@@ -50,7 +50,8 @@ def test_restock_settings_persistence(client, live_server):
             "headers": "",
             "restock_settings-price_change_min": 10,
             "restock_settings-price_change_threshold_percent": 5,
-            'fetch_backend': "html_requests"
+            'fetch_backend': "html_requests",
+            "processor" : 'restock_diff'
         },
         follow_redirects=True
     )
@@ -69,8 +70,7 @@ def test_restock_settings_persistence(client, live_server):
     # This simulates shutting down and restarting the app
     datastore = client.application.config.get('DATASTORE')
     datastore.stop_thread = True
-    datastore.sync_to_json()  # Force write to disk before recreating
-    
+
     # Create a new datastore instance that will read from the saved JSON
     from changedetectionio import store
     new_datastore = store.ChangeDetectionStore(datastore_path="./test-datastore", include_default_watches=False)

changedetectionio/update_worker.py

@@ -527,7 +527,6 @@ class update_worker(threading.Thread):
                         try:
                             self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
 
-
                             # Also save the snapshot on the first time checked, "last checked" will always be updated, so we just check history length.
                             if changed_detected or not watch.history_n:
 
@@ -587,6 +586,7 @@ class update_worker(threading.Thread):
                                                                        'check_count': count
                                                                        })
 
+                    watch.save_data()
 
                 self.current_uuid = None  # Done
                 self.q.task_done()