diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 2a97d7a93..42d4c2e0f 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -932,6 +932,9 @@ def changedetection_app(config=None, datastore_o=None): # Add the flask app secret zipObj.write(os.path.join(datastore_o.datastore_path, "secret.txt"), arcname="secret.txt") + # Add the sqlite3 db + zipObj.write(os.path.join(datastore_o.datastore_path, "watch.db"), arcname="watch.db") + # Add any snapshot data we find, use the full path to access the file, but make the file 'relative' in the Zip. for txt_file_path in Path(datastore_o.datastore_path).rglob('*.txt'): parent_p = txt_file_path.parent diff --git a/changedetectionio/store.py b/changedetectionio/store.py index d4523c4d3..38e8fcbde 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -16,6 +16,7 @@ import sqlite3 from . model import App, Watch + # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods? # Open a github issue if you know something :) # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change @@ -35,9 +36,8 @@ class ChangeDetectionStore: self.json_store_path = "{}/url-watches.json".format(self.datastore_path) self.datastore_path = datastore_path - self.history_db_store_path = "{}/history.db".format(self.datastore_path) #@todo - check for better options - self.history_db_connection = sqlite3.connect(self.history_db_store_path) + self.__history_db_connection = sqlite3.connect("{}/watch.db".format(self.datastore_path)) self.proxy_list = None self.stop_thread = False @@ -77,6 +77,9 @@ class ChangeDetectionStore: if 'application' in from_disk['settings']: self.__data['settings']['application'].update(from_disk['settings']['application']) + # Bump the update version by running updates + self.run_updates() + # Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future. # @todo pretty sure theres a python we todo this with an abstracted(?) object! for uuid, watch in self.__data['watching'].items(): @@ -86,6 +89,7 @@ class ChangeDetectionStore: self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid) print("Watching:", uuid, self.__data['watching'][uuid]['url']) + # First time ran, doesnt exist. except (FileNotFoundError, json.decoder.JSONDecodeError): if include_default_watches: @@ -118,7 +122,6 @@ class ChangeDetectionStore: secret = secrets.token_hex(16) self.__data['settings']['application']['rss_access_token'] = secret - # Proxy list support - available as a selection in settings when text file is imported # CSV list # "name, address", or just "name" @@ -126,8 +129,6 @@ class ChangeDetectionStore: if path.isfile(proxy_list_file): self.import_proxy_list(proxy_list_file) - # Bump the update version by running updates - self.run_updates() self.needs_write = True @@ -137,7 +138,7 @@ class ChangeDetectionStore: # Returns the newest key, but if theres only 1 record, then it's counted as not being new, so return 0. def get_newest_history_key(self, uuid): - cur = self.history_db_connection.cursor() + cur = self.__history_db_connection.cursor() c = cur.execute("SELECT COUNT(*) FROM watch_history WHERE watch_uuid = :uuid", {"uuid": uuid}).fetchone() if c and c[0] <= 1: @@ -146,6 +147,10 @@ class ChangeDetectionStore: max = cur.execute("SELECT MAX(timestamp) FROM watch_history WHERE watch_uuid = :uuid", {"uuid": uuid}).fetchone() return max[0] + def __refresh_history_max_timestamp(self): + # select watch_uuid, max(timestamp) from watch_history group by watch_uuid; + # could be way faster + x=1 def set_last_viewed(self, uuid, timestamp): self.data['watching'][uuid].update({'last_viewed': int(timestamp)}) @@ -190,13 +195,13 @@ class ChangeDetectionStore: def data(self): has_unviewed = False for uuid, v in self.__data['watching'].items(): - self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid) - if int(v['newest_history_key']) <= int(v['last_viewed']): - self.__data['watching'][uuid]['viewed'] = True +# self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid) +# if int(v['newest_history_key']) <= int(v['last_viewed']): +# self.__data['watching'][uuid]['viewed'] = True - else: - self.__data['watching'][uuid]['viewed'] = False - has_unviewed = True +# else: +# self.__data['watching'][uuid]['viewed'] = False +# has_unviewed = True # #106 - Be sure this is None on empty string, False, None, etc # Default var for fetch_backend @@ -508,11 +513,11 @@ class ChangeDetectionStore: - We don't really need this data until we query against it (like for listing other available snapshots in the diff page etc) """ - if self.history_db_connection: + if self.__history_db_connection: # Create the table - self.history_db_connection.execute("CREATE TABLE IF NOT EXISTS watch_history(id INTEGER PRIMARY KEY, watch_uuid VARCHAR(36), timestamp INT, path TEXT, snapshot_type VARCHAR(10))") - self.history_db_connection.execute("CREATE INDEX IF NOT EXISTS `uuid` ON `watch_history` (`watch_uuid`)") - self.history_db_connection.execute("CREATE INDEX IF NOT EXISTS `uuid_timestamp` ON `watch_history` (`watch_uuid`, `timestamp`)") + self.__history_db_connection.execute("CREATE TABLE IF NOT EXISTS watch_history(id INTEGER PRIMARY KEY, watch_uuid VARCHAR(36), timestamp INT, path TEXT, snapshot_type VARCHAR(10))") + self.__history_db_connection.execute("CREATE INDEX IF NOT EXISTS `uuid` ON `watch_history` (`watch_uuid`)") + self.__history_db_connection.execute("CREATE INDEX IF NOT EXISTS `uuid_timestamp` ON `watch_history` (`watch_uuid`, `timestamp`)") # Insert each watch history list as executemany() for faster migration for uuid, watch in self.data['watching'].items(): @@ -524,7 +529,6 @@ class ChangeDetectionStore: history.append((uuid, d, p, 'text')) if len(history): - self.history_db_connection.executemany("INSERT INTO watch_history (watch_uuid, timestamp, path, snapshot_type) VALUES (?,?,?,?)", history) - self.history_db_connection.commit() + self.__history_db_connection.executemany("INSERT INTO watch_history (watch_uuid, timestamp, path, snapshot_type) VALUES (?,?,?,?)", history) + self.__history_db_connection.commit() del(self.data['watching'][uuid]['history']) -