diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index f1b0daec6..54b612a44 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -33,7 +33,6 @@ def sigshutdown_handler(_signo, _stack_frame): global datastore name = signal.Signals(_signo).name logger.critical(f'Shutdown: Got Signal - {name} ({_signo}), Saving DB to disk and calling shutdown') - datastore.sync_to_json() logger.success('Sync JSON to disk complete.') # This will throw a SystemExit exception, because eventlet.wsgi.server doesn't know how to deal with it. # Solution: move to gevent or other server in the future (#2014) diff --git a/changedetectionio/api/api_v1.py b/changedetectionio/api/api_v1.py index 9e03f2e29..e74cc1d50 100644 --- a/changedetectionio/api/api_v1.py +++ b/changedetectionio/api/api_v1.py @@ -126,6 +126,7 @@ class Watch(Resource): return f"Invalid proxy choice, currently supported proxies are '{', '.join(plist)}'", 400 self.datastore.data['watching'][uuid].update(request.json) + self.datastore.data['watching'][uuid].save_data() return "OK", 200 diff --git a/changedetectionio/blueprint/backups/__init__.py b/changedetectionio/blueprint/backups/__init__.py index add44308c..e3710e02c 100644 --- a/changedetectionio/blueprint/backups/__init__.py +++ b/changedetectionio/blueprint/backups/__init__.py @@ -89,8 +89,6 @@ def construct_blueprint(datastore: ChangeDetectionStore): flash("Maximum number of backups reached, please remove some", "error") return redirect(url_for('backups.index')) - # Be sure we're written fresh - datastore.sync_to_json() zip_thread = threading.Thread(target=create_backup, args=(datastore.datastore_path, datastore.data.get("watching"))) zip_thread.start() backup_threads.append(zip_thread) diff --git a/changedetectionio/blueprint/settings/__init__.py b/changedetectionio/blueprint/settings/__init__.py index a8f71567d..360c48fcc 100644 --- a/changedetectionio/blueprint/settings/__init__.py +++ b/changedetectionio/blueprint/settings/__init__.py @@ -71,12 +71,12 @@ def construct_blueprint(datastore: ChangeDetectionStore): if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password): datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password - datastore.needs_write_urgent = True + datastore.save_settings() flash("Password protection enabled.", 'notice') flask_login.logout_user() return redirect(url_for('index')) - datastore.needs_write_urgent = True + datastore.save_settings() flash("Settings updated.") else: @@ -100,8 +100,7 @@ def construct_blueprint(datastore: ChangeDetectionStore): datastore.data['settings']['application'].update(app_update) datastore.data['settings']['requests'].update(form.data['requests']) - - datastore.needs_write_urgent = True + datastore.save_settings() flash("Settings updated.") output = render_template("settings.html", @@ -125,7 +124,6 @@ def construct_blueprint(datastore: ChangeDetectionStore): def settings_reset_api_key(): secret = secrets.token_hex(16) datastore.data['settings']['application']['api_access_token'] = secret - datastore.needs_write_urgent = True flash("API Key was regenerated.") return redirect(url_for('settings.settings_page')+'#api') diff --git a/changedetectionio/blueprint/tags/__init__.py b/changedetectionio/blueprint/tags/__init__.py index d7086213c..f580503a7 100644 --- a/changedetectionio/blueprint/tags/__init__.py +++ b/changedetectionio/blueprint/tags/__init__.py @@ -56,6 +56,7 @@ def construct_blueprint(datastore: ChangeDetectionStore): def mute(uuid): if datastore.data['settings']['application']['tags'].get(uuid): datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = not datastore.data['settings']['application']['tags'][uuid]['notification_muted'] + datastore.data['settings']['application']['tags'][uuid].save_data() return redirect(url_for('tags.tags_overview_page')) @tags_blueprint.route("/delete/", methods=['GET']) @@ -176,7 +177,8 @@ def construct_blueprint(datastore: ChangeDetectionStore): datastore.data['settings']['application']['tags'][uuid].update(form.data) datastore.data['settings']['application']['tags'][uuid]['processor'] = 'restock_diff' - datastore.needs_write_urgent = True + datastore.data['settings']['application']['tags'][uuid].save_data() + flash("Updated") return redirect(url_for('tags.tags_overview_page')) diff --git a/changedetectionio/blueprint/ui/__init__.py b/changedetectionio/blueprint/ui/__init__.py index 16f07f379..c320702a5 100644 --- a/changedetectionio/blueprint/ui/__init__.py +++ b/changedetectionio/blueprint/ui/__init__.py @@ -163,6 +163,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat uuid = uuid.strip() if datastore.data['watching'].get(uuid): datastore.data['watching'][uuid.strip()]['paused'] = True + datastore.data['watching'][uuid.strip()].save_data() flash("{} watches paused".format(len(uuids))) elif (op == 'unpause'): @@ -170,6 +171,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat uuid = uuid.strip() if datastore.data['watching'].get(uuid): datastore.data['watching'][uuid.strip()]['paused'] = False + datastore.data['watching'][uuid.strip()].save_data() flash("{} watches unpaused".format(len(uuids))) elif (op == 'mark-viewed'): @@ -184,6 +186,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat uuid = uuid.strip() if datastore.data['watching'].get(uuid): datastore.data['watching'][uuid.strip()]['notification_muted'] = True + datastore.data['watching'][uuid.strip()].save_data() flash("{} watches muted".format(len(uuids))) elif (op == 'unmute'): @@ -191,6 +194,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat uuid = uuid.strip() if datastore.data['watching'].get(uuid): datastore.data['watching'][uuid.strip()]['notification_muted'] = False + datastore.data['watching'][uuid.strip()].save_data() flash("{} watches un-muted".format(len(uuids))) elif (op == 'recheck'): @@ -206,6 +210,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat uuid = uuid.strip() if datastore.data['watching'].get(uuid): datastore.data['watching'][uuid]["last_error"] = False + datastore.data['watching'][uuid].save_data() flash(f"{len(uuids)} watches errors cleared") elif (op == 'clear-history'): @@ -244,6 +249,9 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_updat flash(f"{len(uuids)} watches were tagged") + for uuid in uuids: + datastore.data['watching'][uuid.strip()].save_data() + return redirect(url_for('index')) diff --git a/changedetectionio/blueprint/ui/edit.py b/changedetectionio/blueprint/ui/edit.py index 3a9d45326..487084043 100644 --- a/changedetectionio/blueprint/ui/edit.py +++ b/changedetectionio/blueprint/ui/edit.py @@ -49,8 +49,8 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe datastore.clear_watch_history(uuid) redirect(url_for('ui_edit.edit_page', uuid=uuid)) - # be sure we update with a copy instead of accidently editing the live object by reference - default = deepcopy(datastore.data['watching'][uuid]) + + default = datastore.data['watching'][uuid] # Defaults for proxy choice if datastore.proxy_list is not None: # When enabled @@ -114,10 +114,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe extra_update_obj['paused'] = False extra_update_obj['time_between_check'] = form.time_between_check.data - - # Ignore text - form_ignore_text = form.ignore_text.data - datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text + extra_update_obj['ignore_text'] = form.ignore_text.data # Be sure proxy value is None if datastore.proxy_list is not None and form.data['proxy'] == '': @@ -143,23 +140,23 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe tag_uuids.append(datastore.add_tag(name=t)) extra_update_obj['tags'] = tag_uuids - datastore.data['watching'][uuid].update(form.data) - datastore.data['watching'][uuid].update(extra_update_obj) + + if not datastore.data['watching'][uuid].get('tags'): # Force it to be a list, because form.data['tags'] will be string if nothing found # And del(form.data['tags'] ) wont work either for some reason datastore.data['watching'][uuid]['tags'] = [] + datastore.update_watch(uuid=uuid, update_obj=form.data | extra_update_obj) + # Recast it if need be to right data Watch handler - processor_name = form.data.get('processor') + processor_name = datastore.data['watching'][uuid].get('processor') watch_class = processors.get_watch_model_for_processor(processor_name) datastore.data['watching'][uuid] = watch_class(datastore_path=datastore.datastore_path, default=datastore.data['watching'][uuid]) + datastore.data['watching'][uuid].save_data() flash("Updated watch - unpaused!" if request.args.get('unpause_on_save') else "Updated watch.") - # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds - # But in the case something is added we should save straight away - datastore.needs_write_urgent = True # Do not queue on edit if its not within the time range @@ -186,6 +183,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}") return False + ############################# if not datastore.data['watching'][uuid].get('paused') and is_in_schedule: # Queue the watch for immediate recheck, with a higher priority diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index f4d9f0dff..6ba1461db 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -370,7 +370,6 @@ def changedetection_app(config=None, datastore_o=None): elif op == 'mute': datastore.data['watching'][uuid].toggle_mute() - datastore.needs_write = True return redirect(url_for('index', tag = active_tag_uuid)) # Sort by last_changed and add the uuid which is usually the key.. diff --git a/changedetectionio/model/Tag.py b/changedetectionio/model/Tag.py index 6dca480cd..7eb09b821 100644 --- a/changedetectionio/model/Tag.py +++ b/changedetectionio/model/Tag.py @@ -1,14 +1,57 @@ -from changedetectionio.model import watch_base +import os +import json +import uuid as uuid_builder +import time +from copy import deepcopy +from loguru import logger + +from changedetectionio.model import watch_base, schema class model(watch_base): + """Tag model that writes to tags/{uuid}/tag.json instead of the main watch directory""" + __datastore_path = None def __init__(self, *arg, **kw): super(model, self).__init__(*arg, **kw) + self.__datastore_path = kw.get("datastore_path") self['overrides_watch'] = kw.get('default', {}).get('overrides_watch') if kw.get('default'): self.update(kw['default']) del kw['default'] + + @property + def watch_data_dir(self): + # Override to use tags directory instead of the normal watch data directory + datastore_path = getattr(self, '_model__datastore_path', None) + if datastore_path: + tags_path = os.path.join(datastore_path, 'tags') + # Make sure the tags directory exists + if not os.path.exists(tags_path): + os.makedirs(tags_path) + return os.path.join(tags_path, self['uuid']) + return None + + def save_data(self): + """Override to save tag to tags/{uuid}/tag.json""" + logger.debug(f"Saving tag {self['uuid']}") + + if not self.get('uuid'): + # Might have been called when creating the tag + return + + tags_path = os.path.join(self.__datastore_path, 'tags') + if not os.path.isdir(tags_path): + os.mkdir(os.path.join(tags_path)) + + path = os.path.join(tags_path, self.get('uuid')+".json") + try: + with open(path + ".tmp", 'w') as json_file: + json.dump(self.get_data(), json_file, indent=4) + os.replace(path + ".tmp", path) + except Exception as e: + logger.error(f"Error writing JSON for tag {self.get('uuid')}!! (JSON file save was skipped) : {str(e)}") + diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index f91371c57..f5b1173bc 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -38,18 +38,13 @@ class model(watch_base): jitter_seconds = 0 def __init__(self, *arg, **kw): - self.__datastore_path = kw.get('datastore_path') - if kw.get('datastore_path'): - del kw['datastore_path'] + super(model, self).__init__(*arg, **kw) if kw.get('default'): self.update(kw['default']) del kw['default'] - if self.get('default'): - del self['default'] - # Be sure the cached timestamp is ready bump = self.history @@ -301,6 +296,7 @@ class model(watch_base): # result_obj from fetch_site_status.run() def save_history_text(self, contents, timestamp, snapshot_id): import brotli + import tempfile logger.trace(f"{self.get('uuid')} - Updating history.txt with timestamp {timestamp}") @@ -417,11 +413,6 @@ class model(watch_base): def snapshot_error_screenshot_ctime(self): return self.__get_file_ctime('last-error-screenshot.png') - @property - def watch_data_dir(self): - # The base dir of the watch data - return os.path.join(self.__datastore_path, self['uuid']) if self.__datastore_path else None - def get_error_text(self): """Return the text saved from a previous request that resulted in a non-200 error""" fname = os.path.join(self.watch_data_dir, "last-error.txt") diff --git a/changedetectionio/model/__init__.py b/changedetectionio/model/__init__.py index 6684c4b0c..ddcba2424 100644 --- a/changedetectionio/model/__init__.py +++ b/changedetectionio/model/__init__.py @@ -1,7 +1,9 @@ import os import uuid from copy import deepcopy +from loguru import logger import time +import json from changedetectionio import strtobool from changedetectionio.notification import default_notification_format_for_watch @@ -49,6 +51,7 @@ schema = { 'previous_md5': False, 'previous_md5_before_filters': False, # Used for skipping changedetection entirely 'processor': 'text_json_diff', # could be restock_diff or others from .processors + 'processor_state': {}, # Extra configs for custom processors/plugins, keyed by processor name 'price_change_threshold_percent': None, 'proxy': None, # Preferred proxy connection 'remote_server_reply': None, # From 'server' reply header @@ -131,12 +134,14 @@ schema = { class watch_base(dict): __data = {} + __datastore_path = None + __save_enabled = True def __init__(self, *arg, **kw): # Initialize internal data storage self.__data = deepcopy(schema) - + self.__datastore_path = kw.pop('datastore_path', None) # Initialize as empty dict but maintain dict interface super(watch_base, self).__init__() @@ -147,7 +152,18 @@ class watch_base(dict): # Generate UUID if needed if not self.__data.get('uuid'): self.__data['uuid'] = str(uuid.uuid4()) - + + if self.__data.get('default'): + del(self.__data['default']) + + @property + def watch_data_dir(self): + # The base dir of the watch data + return os.path.join(self.__datastore_path, self['uuid']) if self.__datastore_path else None + + def enable_saving(self): + self.__save_enabled = True + # Dictionary interface methods to use self.__data def __getitem__(self, key): return self.__data[key] @@ -155,7 +171,7 @@ class watch_base(dict): def __setitem__(self, key, value): self.__data[key] = value self.__data['last_modified'] = time.time() - + def __delitem__(self, key): del self.__data[key] @@ -205,3 +221,26 @@ class watch_base(dict): def get_data(self): """Returns the internal data dictionary""" return self.__data + + def save_data(self): + if self.__save_enabled: + if not self.__data.get('uuid'): + # Might have been called when creating the watch + return + + logger.debug(f"Saving watch {self['uuid']}") + path = os.path.join(self.__datastore_path, self.get('uuid')) + filepath = os.path.join(str(path), "watch.json") + if not os.path.exists(path): + os.mkdir(path) + + try: + import tempfile + with tempfile.NamedTemporaryFile(mode='wb+', delete=False) as tmp: + tmp.write(json.dumps(self.get_data(), indent=2).encode('utf-8')) + tmp.flush() + os.replace(tmp.name, filepath) + + + except Exception as e: + logger.error(f"Error writing JSON for {self.get('uuid')}!! (JSON file save was skipped) : {str(e)}") diff --git a/changedetectionio/processors/restock_diff/__init__.py b/changedetectionio/processors/restock_diff/__init__.py index c02813e9b..2229a5b6c 100644 --- a/changedetectionio/processors/restock_diff/__init__.py +++ b/changedetectionio/processors/restock_diff/__init__.py @@ -56,8 +56,13 @@ class Restock(dict): super().__setitem__(key, value) class Watch(BaseWatch): + + def load_extra_vars(self): + # something from disk? + def __init__(self, *arg, **kw): super().__init__(*arg, **kw) + # Restock Obj helps with the state of the situation self['restock'] = Restock(kw['default']['restock']) if kw.get('default') and kw['default'].get('restock') else Restock() self['restock_settings'] = kw['default']['restock_settings'] if kw.get('default',{}).get('restock_settings') else { diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 3a2b1f292..0c9ec5ca1 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -31,11 +31,6 @@ dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ]) # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change class ChangeDetectionStore: lock = Lock() - # For general updates/writes that can wait a few seconds - needs_write = False - - # For when we edit, we should write to disk - needs_write_urgent = False __version_check = True @@ -46,7 +41,7 @@ class ChangeDetectionStore: self.datastore_path = datastore_path self.json_store_path = "{}/url-watches.json".format(self.datastore_path) logger.info(f"Datastore path is '{self.json_store_path}'") - self.needs_write = False + self.start_time = time.time() self.stop_thread = False @@ -56,39 +51,30 @@ class ChangeDetectionStore: # So when someone gives us a backup file to examine, we know exactly what code they were running. self.__data['build_sha'] = f.read() + self.generic_definition = deepcopy(Watch.model(datastore_path = datastore_path, default={})) + try: - # @todo retest with ", encoding='utf-8'" - with open(self.json_store_path) as json_file: - from_disk = json.load(json_file) + import os + # First load global settings from the main JSON file if it exists + if os.path.isfile(self.json_store_path): + with open(self.json_store_path) as json_file: + from_disk = json.load(json_file) + + # Load app_guid and settings from the main JSON file + if 'app_guid' in from_disk: + self.__data['app_guid'] = from_disk['app_guid'] + + if 'settings' in from_disk: + if 'headers' in from_disk['settings']: + self.__data['settings']['headers'].update(from_disk['settings']['headers']) + + if 'requests' in from_disk['settings']: + self.__data['settings']['requests'].update(from_disk['settings']['requests']) + + if 'application' in from_disk['settings']: + self.__data['settings']['application'].update(from_disk['settings']['application']) - # @todo isnt there a way todo this dict.update recursively? - # Problem here is if the one on the disk is missing a sub-struct, it wont be present anymore. - if 'watching' in from_disk: - self.__data['watching'].update(from_disk['watching']) - if 'app_guid' in from_disk: - self.__data['app_guid'] = from_disk['app_guid'] - - if 'settings' in from_disk: - if 'headers' in from_disk['settings']: - self.__data['settings']['headers'].update(from_disk['settings']['headers']) - - if 'requests' in from_disk['settings']: - self.__data['settings']['requests'].update(from_disk['settings']['requests']) - - if 'application' in from_disk['settings']: - self.__data['settings']['application'].update(from_disk['settings']['application']) - - # Convert each existing watch back to the Watch.model object - for uuid, watch in self.__data['watching'].items(): - self.__data['watching'][uuid] = self.rehydrate_entity(default_dict=watch) - logger.info(f"Watching: {uuid} {watch['url']}") - - # And for Tags also, should be Restock type because it has extra settings - # @todo make this smarter! - for uuid, tag in self.__data['settings']['application']['tags'].items(): - self.__data['settings']['application']['tags'][uuid] = self.rehydrate_entity(default_dict=tag, processor_override='restock_diff') - logger.info(f"Tag: {uuid} {tag['title']}") # First time ran, Create the datastore. except (FileNotFoundError): @@ -107,6 +93,8 @@ class ChangeDetectionStore: else: # Bump the update version by running updates + self.scan_load_watches() + self.scan_load_tags() self.run_updates() self.__data['version_tag'] = version_tag @@ -138,10 +126,53 @@ class ChangeDetectionStore: secret = secrets.token_hex(16) self.__data['settings']['application']['api_access_token'] = secret - self.needs_write = True + def scan_load_watches(self): - # Finally start the thread that will manage periodic data saves to JSON - save_data_thread = threading.Thread(target=self.save_datastore).start() + # Now scan for individual watch.json files in the datastore directory + import pathlib + watch_jsons = list(pathlib.Path(self.datastore_path).rglob("*/watch.json")) + + for watch_file in watch_jsons: + # Extract UUID from the directory name (parent directory of watch.json) + uuid = watch_file.parent.name + + try: + with open(watch_file, 'r') as f: + watch_data = json.load(f) + # Create a Watch object and add it to the datastore + self.__data['watching'][uuid] = self.rehydrate_entity(default_dict=watch_data) + logger.info(f"Watching: {uuid} {watch_data.get('url')}") + + except Exception as e: + logger.error(f"Error loading watch from {watch_file}: {str(e)}") + continue + logger.debug(f"{len(self.__data['watching'])} watches loaded.") + + def scan_load_tags(self): + import pathlib + # Now scan for individual tag.json files in the tags directory + tags_path = os.path.join(self.datastore_path, 'tags') + if os.path.exists(tags_path): + tag_jsons = list(pathlib.Path(tags_path).rglob("*.json")) + + for tag_file in tag_jsons: + # Extract UUID from the directory name (parent directory of tag.json) + + try: + with open(tag_file, 'r') as f: + tag_data = json.load(f) + uuid = str(tag_file).replace('.json', '') + tag_data['uuid'] = uuid + # Create a Tag object and add it to the datastore + self.__data['settings']['application']['tags'][uuid] = self.rehydrate_entity( + default_dict=tag_data, + processor_override='restock_diff' + ) + logger.info(f"Tag: {uuid} {tag_data.get('title', 'No title found')}") + except Exception as e: + logger.error(f"Error loading tag from {tag_file}: {str(e)}") + continue + logger.debug(f"{len(self.__data['settings']['application']['tags'])} tags loaded.") def rehydrate_entity(self, default_dict: dict, processor_override=None): @@ -152,16 +183,17 @@ class ChangeDetectionStore: watch_class = get_watch_model_for_processor(processor_override) default_dict['processor'] = processor_override entity = watch_class(datastore_path=self.datastore_path, default=default_dict) + entity.enable_saving() return entity def set_last_viewed(self, uuid, timestamp): logger.debug(f"Setting watch UUID: {uuid} last viewed to {int(timestamp)}") self.data['watching'][uuid].update({'last_viewed': int(timestamp)}) - self.needs_write = True + self.data['watching'][uuid].save_data() def remove_password(self): self.__data['settings']['application']['password'] = False - self.needs_write = True + self.save_settings() def update_watch(self, uuid, update_obj): """ @@ -171,21 +203,16 @@ class ChangeDetectionStore: if not uuid in self.data['watching'].keys() or update_obj is None: return - with self.lock: - # Make sure we're working with a proper Watch object - watch = self.data['watching'].get(uuid) - - # Handle None values - they mean "delete this key" - keys_to_remove = [k for k, v in update_obj.items() if v is None] - for k in keys_to_remove: - if k in watch: - del watch[k] - del update_obj[k] - - # Deep merge with the rest - always_merger.merge(watch, update_obj) - - self.needs_write = True + # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures... + for dict_key, d in self.generic_definition.items(): + if isinstance(d, dict): + if update_obj is not None and dict_key in update_obj: + self.__data['watching'][uuid][dict_key].update(update_obj[dict_key]) + del (update_obj[dict_key]) + + self.__data['watching'][uuid].update(update_obj) + self.__data['watching'][uuid].save_data() + @property def threshold_seconds(self): @@ -245,8 +272,6 @@ class ChangeDetectionStore: shutil.rmtree(path) del self.data['watching'][uuid] - self.needs_write_urgent = True - # Clone a watch by UUID def clone(self, uuid): url = self.data['watching'][uuid].get('url') @@ -266,7 +291,6 @@ class ChangeDetectionStore: # Remove a watchs data but keep the entry (URL etc) def clear_watch_history(self, uuid): self.__data['watching'][uuid].clear_watch() - self.needs_write_urgent = True def add_watch(self, url, tag='', extras=None, tag_uuids=None, write_to_disk_now=True): import requests @@ -357,15 +381,11 @@ class ChangeDetectionStore: if not apply_extras.get('date_created'): apply_extras['date_created'] = int(time.time()) - - new_watch.update(apply_extras) new_watch.ensure_data_dir_exists() + new_watch.update(apply_extras) + self.__data['watching'][new_uuid] = new_watch - - - if write_to_disk_now: - self.sync_to_json() - + self.__data['watching'][new_uuid].save_data() logger.debug(f"Added '{url}'") return new_uuid @@ -379,58 +399,22 @@ class ChangeDetectionStore: return False - def sync_to_json(self): - logger.info("Saving JSON..") + def save_settings(self): + logger.info("Saving application settings...") try: - data = deepcopy(self.__data) - except RuntimeError as e: - # Try again in 15 seconds - time.sleep(15) - logger.error(f"! Data changed when writing to JSON, trying again.. {str(e)}") - self.sync_to_json() - return - else: - - try: - # Re #286 - First write to a temp file, then confirm it looks OK and rename it - # This is a fairly basic strategy to deal with the case that the file is corrupted, - # system was out of memory, out of RAM etc - with open(self.json_store_path+".tmp", 'w') as json_file: - json.dump(data, json_file, indent=4) - os.replace(self.json_store_path+".tmp", self.json_store_path) - except Exception as e: - logger.error(f"Error writing JSON!! (Main JSON file save was skipped) : {str(e)}") - - self.needs_write = False - self.needs_write_urgent = False - - # Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON - # by just running periodically in one thread, according to python, dict updates are threadsafe. - def save_datastore(self): - - while True: - if self.stop_thread: - # Suppressing "Logging error in Loguru Handler #0" during CICD. - # Not a meaningful difference for a real use-case just for CICD. - # the side effect is a "Shutting down datastore thread" message - # at the end of each test. - # But still more looking better. - import sys - logger.remove() - logger.add(sys.stderr) - - logger.info("Shutting down datastore thread") - return - - if self.needs_write or self.needs_write_urgent: - self.sync_to_json() - - # Once per minute is enough, more and it can cause high CPU usage - # better here is to use something like self.app.config.exit.wait(1), but we cant get to 'app' from here - for i in range(120): - time.sleep(0.5) - if self.stop_thread or self.needs_write_urgent: - break + # Only save app settings, not the watches or tags (they're saved individually) + data = {'settings': self.__data.get('settings')} + #data = deepcopy(self.__data) + + # Remove the watches from the main JSON file + if 'watching' in data: + del data['watching'] + + # Remove the tags from the main JSON file since they're saved individually now +# if 'settings' in data and 'application' in data['settings'] and 'tags' in data['settings']['application']: +# del data['settings']['application']['tags'] + except Exception as e: + x=1 # Go through the datastore path and remove any snapshots that are not mentioned in the index # This usually is not used, but can be handy. @@ -584,16 +568,17 @@ class ChangeDetectionStore: # Eventually almost everything todo with a watch will apply as a Tag # So we use the same model as a Watch - with self.lock: - from .model import Tag - new_tag = Tag.model(datastore_path=self.datastore_path, default={ - 'title': name.strip(), - 'date_created': int(time.time()) - }) + from .model import Tag + new_tag = Tag.model(datastore_path=self.datastore_path, default={ + 'title': name.strip(), + 'date_created': int(time.time()) + }) - new_uuid = new_tag.get('uuid') + new_uuid = new_tag.get('uuid') + + self.__data['settings']['application']['tags'][new_uuid] = new_tag + self.__data['settings']['application']['tags'][new_uuid].save_data() - self.__data['settings']['application']['tags'][new_uuid] = new_tag return new_uuid diff --git a/changedetectionio/tests/test_conditions.py b/changedetectionio/tests/test_conditions.py index 2f3637fe4..51dde1449 100644 --- a/changedetectionio/tests/test_conditions.py +++ b/changedetectionio/tests/test_conditions.py @@ -50,7 +50,7 @@ def test_conditions_with_text_and_number(client, live_server): """Test that both text and number conditions work together with AND logic.""" set_original_response("50") -# live_server_setup(live_server) + #live_server_setup(live_server) test_url = url_for('test_endpoint', _external=True) diff --git a/changedetectionio/tests/test_restock_save_load_settings.py b/changedetectionio/tests/test_restock_save_load_settings.py index de56f38b0..8c8685e2a 100644 --- a/changedetectionio/tests/test_restock_save_load_settings.py +++ b/changedetectionio/tests/test_restock_save_load_settings.py @@ -50,7 +50,8 @@ def test_restock_settings_persistence(client, live_server): "headers": "", "restock_settings-price_change_min": 10, "restock_settings-price_change_threshold_percent": 5, - 'fetch_backend': "html_requests" + 'fetch_backend': "html_requests", + "processor" : 'restock_diff' }, follow_redirects=True ) @@ -69,8 +70,7 @@ def test_restock_settings_persistence(client, live_server): # This simulates shutting down and restarting the app datastore = client.application.config.get('DATASTORE') datastore.stop_thread = True - datastore.sync_to_json() # Force write to disk before recreating - + # Create a new datastore instance that will read from the saved JSON from changedetectionio import store new_datastore = store.ChangeDetectionStore(datastore_path="./test-datastore", include_default_watches=False) diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 052e449a7..83902f856 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -527,7 +527,6 @@ class update_worker(threading.Thread): try: self.datastore.update_watch(uuid=uuid, update_obj=update_obj) - # Also save the snapshot on the first time checked, "last checked" will always be updated, so we just check history length. if changed_detected or not watch.history_n: @@ -587,6 +586,7 @@ class update_worker(threading.Thread): 'check_count': count }) + watch.save_data() self.current_uuid = None # Done self.q.task_done()