Use a history index per processor, no need to reset history when switching modes

This commit is contained in:
dgtlmoon committed 2025-12-29 16:22:13 +01:00
1 parent 8773ee813a
commit a6c849b14b
6 files changed
+37 -20

No files matched your search

+1 -1
View File
@@ -344,7 +344,7 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
fetch_start_time += 1
await asyncio.sleep(1)
watch.save_history_text(contents=contents,
watch.save_history_blob(contents=contents,
timestamp=int(fetch_start_time),
snapshot_id=update_obj.get('previous_md5', 'none'))
-5
View File
@@ -133,11 +133,6 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
if request.method == 'POST' and form.validate():
# If they changed processor, it makes sense to reset it.
if datastore.data['watching'][uuid].get('processor') != form.data.get('processor'):
datastore.data['watching'][uuid].clear_watch()
flash("Reset watch history due to change of processor")
extra_update_obj = {
'consecutive_filter_failures': 0,
'last_error' : False
+13 -5
View File
@@ -223,6 +223,14 @@ class model(watch_base):
domain = parsed.hostname
return domain
@property
def history_index_filename(self):
# So that you dont try to view different histories in different 'diff' setups, can confuse cdio.
if self.get('processor') in ['text_json_diff']:
return 'history.txt'
else:
return f'history-{self.get('processor')}.txt'
def clear_watch(self):
import pathlib
@@ -324,7 +332,7 @@ class model(watch_base):
return []
# Read the history file as a dict
fname = os.path.join(self.watch_data_dir, "history.txt")
fname = os.path.join(self.watch_data_dir, self.history_index_filename)
if os.path.isfile(fname):
logger.debug(f"Reading watch history index for {self.get('uuid')}")
with open(fname, "r", encoding='utf-8') as f:
@@ -360,7 +368,7 @@ class model(watch_base):
@property
def has_history(self):
fname = os.path.join(self.watch_data_dir, "history.txt")
fname = os.path.join(self.watch_data_dir, self.history_index_filename)
return os.path.isfile(fname)
@property
@@ -476,9 +484,9 @@ class model(watch_base):
# Save some text file to the appropriate path and bump the history
# result_obj from fetch_site_status.run()
def save_history_text(self, contents, timestamp, snapshot_id):
def save_history_blob(self, contents, timestamp, snapshot_id):
logger.trace(f"{self.get('uuid')} - Updating history.txt with timestamp {timestamp}")
logger.trace(f"{self.get('uuid')} - Updating {self.history_index_filename} with timestamp {timestamp}")
self.ensure_data_dir_exists()
@@ -530,7 +538,7 @@ class model(watch_base):
self._write_atomic(dest, contents.encode('utf-8'))
# Append to history.txt atomically
index_fname = os.path.join(self.watch_data_dir, "history.txt")
index_fname = os.path.join(self.watch_data_dir, self.history_index_filename)
index_line = f"{timestamp},{snapshot_fname}\n"
with open(index_fname, 'a', encoding='utf-8') as f:
+14
View File
@@ -1,3 +1,5 @@
import shutil
from changedetectionio.strtobool import strtobool
from changedetectionio.validate_url import is_safe_valid_url
@@ -1122,3 +1124,15 @@ class ChangeDetectionStore:
else:
# safe fallback to text
self.data['settings']['application']['rss_content_format'] = RSS_CONTENT_FORMAT_DEFAULT
# Different processors now hold their own history.txt
def update_25(self):
for uuid, watch in self.data['watching'].items():
processor = self.data['watching'][uuid].get('processor')
if processor != 'text_json_diff':
old_history_txt = os.path.join(self.datastore_path, "history.txt")
target_history_name = f"history-{processor}.txt"
if os.path.isfile(old_history_txt) and not os.path.isfile(target_history_name):
new_history_txt = os.path.join(self.datastore_path, target_history_name)
logger.debug(f"Renaming history index {old_history_txt} to {new_history_txt}...")
shutil.move(old_history_txt, new_history_txt)
@@ -38,7 +38,7 @@ class TestTriggerConditions(unittest.TestCase):
first_content = "I saw 100 people at a rock show"
timestamp1 = int(time.time())
snapshot_id1 = str(uuid.uuid4())
watch.save_history_text(contents=first_content,
watch.save_history_blob(contents=first_content,
timestamp=timestamp1,
snapshot_id=snapshot_id1)
@@ -46,7 +46,7 @@ class TestTriggerConditions(unittest.TestCase):
second_content = "I saw 200 people at a rock show"
timestamp2 = int(time.time()) + 60
snapshot_id2 = str(uuid.uuid4())
watch.save_history_text(contents=second_content,
watch.save_history_blob(contents=second_content,
timestamp=timestamp2,
snapshot_id=snapshot_id2)
@@ -18,12 +18,12 @@ class TestDiffBuilder(unittest.TestCase):
# Contents from the browser are always returned from the browser/requests/etc as str, str is basically UTF-16 in python
watch.save_history_text(contents="hello world", timestamp=100, snapshot_id=str(uuid_builder.uuid4()))
watch.save_history_text(contents="hello world", timestamp=105, snapshot_id=str(uuid_builder.uuid4()))
watch.save_history_text(contents="hello world", timestamp=109, snapshot_id=str(uuid_builder.uuid4()))
watch.save_history_text(contents="hello world", timestamp=112, snapshot_id=str(uuid_builder.uuid4()))
watch.save_history_text(contents="hello world", timestamp=115, snapshot_id=str(uuid_builder.uuid4()))
watch.save_history_text(contents="hello world", timestamp=117, snapshot_id=str(uuid_builder.uuid4()))
watch.save_history_blob(contents="hello world", timestamp=100, snapshot_id=str(uuid_builder.uuid4()))
watch.save_history_blob(contents="hello world", timestamp=105, snapshot_id=str(uuid_builder.uuid4()))
watch.save_history_blob(contents="hello world", timestamp=109, snapshot_id=str(uuid_builder.uuid4()))
watch.save_history_blob(contents="hello world", timestamp=112, snapshot_id=str(uuid_builder.uuid4()))
watch.save_history_blob(contents="hello world", timestamp=115, snapshot_id=str(uuid_builder.uuid4()))
watch.save_history_blob(contents="hello world", timestamp=117, snapshot_id=str(uuid_builder.uuid4()))
p = watch.get_from_version_based_on_last_viewed
assert p == "100", "Correct 'last viewed' timestamp was detected"
@@ -53,7 +53,7 @@ class TestDiffBuilder(unittest.TestCase):
p = watch.get_from_version_based_on_last_viewed
assert p == None, "None when no history available"
watch.save_history_text(contents="hello world", timestamp=100, snapshot_id=str(uuid_builder.uuid4()))
watch.save_history_blob(contents="hello world", timestamp=100, snapshot_id=str(uuid_builder.uuid4()))
p = watch.get_from_version_based_on_last_viewed
assert p == "100", "Correct with only one history snapshot"