mirror of
https://github.com/dgtlmoon/changedetection.io.git
synced 2026-11-17 03:16:59 +00:00
Use a history index per processor, no need to reset history when switching modes
This commit is contained in:
1 parent
8773ee813a
commit
a6c849b14b
6 files changed
+37
-20
No files matched your search
@@ -344,7 +344,7 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
|
||||
fetch_start_time += 1
|
||||
await asyncio.sleep(1)
|
||||
|
||||
watch.save_history_text(contents=contents,
|
||||
watch.save_history_blob(contents=contents,
|
||||
timestamp=int(fetch_start_time),
|
||||
snapshot_id=update_obj.get('previous_md5', 'none'))
|
||||
|
||||
|
||||
@@ -133,11 +133,6 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
|
||||
|
||||
if request.method == 'POST' and form.validate():
|
||||
|
||||
# If they changed processor, it makes sense to reset it.
|
||||
if datastore.data['watching'][uuid].get('processor') != form.data.get('processor'):
|
||||
datastore.data['watching'][uuid].clear_watch()
|
||||
flash("Reset watch history due to change of processor")
|
||||
|
||||
extra_update_obj = {
|
||||
'consecutive_filter_failures': 0,
|
||||
'last_error' : False
|
||||
|
||||
@@ -223,6 +223,14 @@ class model(watch_base):
|
||||
domain = parsed.hostname
|
||||
return domain
|
||||
|
||||
@property
|
||||
def history_index_filename(self):
|
||||
# So that you dont try to view different histories in different 'diff' setups, can confuse cdio.
|
||||
if self.get('processor') in ['text_json_diff']:
|
||||
return 'history.txt'
|
||||
else:
|
||||
return f'history-{self.get('processor')}.txt'
|
||||
|
||||
def clear_watch(self):
|
||||
import pathlib
|
||||
|
||||
@@ -324,7 +332,7 @@ class model(watch_base):
|
||||
return []
|
||||
|
||||
# Read the history file as a dict
|
||||
fname = os.path.join(self.watch_data_dir, "history.txt")
|
||||
fname = os.path.join(self.watch_data_dir, self.history_index_filename)
|
||||
if os.path.isfile(fname):
|
||||
logger.debug(f"Reading watch history index for {self.get('uuid')}")
|
||||
with open(fname, "r", encoding='utf-8') as f:
|
||||
@@ -360,7 +368,7 @@ class model(watch_base):
|
||||
|
||||
@property
|
||||
def has_history(self):
|
||||
fname = os.path.join(self.watch_data_dir, "history.txt")
|
||||
fname = os.path.join(self.watch_data_dir, self.history_index_filename)
|
||||
return os.path.isfile(fname)
|
||||
|
||||
@property
|
||||
@@ -476,9 +484,9 @@ class model(watch_base):
|
||||
|
||||
# Save some text file to the appropriate path and bump the history
|
||||
# result_obj from fetch_site_status.run()
|
||||
def save_history_text(self, contents, timestamp, snapshot_id):
|
||||
def save_history_blob(self, contents, timestamp, snapshot_id):
|
||||
|
||||
logger.trace(f"{self.get('uuid')} - Updating history.txt with timestamp {timestamp}")
|
||||
logger.trace(f"{self.get('uuid')} - Updating {self.history_index_filename} with timestamp {timestamp}")
|
||||
|
||||
self.ensure_data_dir_exists()
|
||||
|
||||
@@ -530,7 +538,7 @@ class model(watch_base):
|
||||
self._write_atomic(dest, contents.encode('utf-8'))
|
||||
|
||||
# Append to history.txt atomically
|
||||
index_fname = os.path.join(self.watch_data_dir, "history.txt")
|
||||
index_fname = os.path.join(self.watch_data_dir, self.history_index_filename)
|
||||
index_line = f"{timestamp},{snapshot_fname}\n"
|
||||
|
||||
with open(index_fname, 'a', encoding='utf-8') as f:
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import shutil
|
||||
|
||||
from changedetectionio.strtobool import strtobool
|
||||
|
||||
from changedetectionio.validate_url import is_safe_valid_url
|
||||
@@ -1122,3 +1124,15 @@ class ChangeDetectionStore:
|
||||
else:
|
||||
# safe fallback to text
|
||||
self.data['settings']['application']['rss_content_format'] = RSS_CONTENT_FORMAT_DEFAULT
|
||||
|
||||
# Different processors now hold their own history.txt
|
||||
def update_25(self):
|
||||
for uuid, watch in self.data['watching'].items():
|
||||
processor = self.data['watching'][uuid].get('processor')
|
||||
if processor != 'text_json_diff':
|
||||
old_history_txt = os.path.join(self.datastore_path, "history.txt")
|
||||
target_history_name = f"history-{processor}.txt"
|
||||
if os.path.isfile(old_history_txt) and not os.path.isfile(target_history_name):
|
||||
new_history_txt = os.path.join(self.datastore_path, target_history_name)
|
||||
logger.debug(f"Renaming history index {old_history_txt} to {new_history_txt}...")
|
||||
shutil.move(old_history_txt, new_history_txt)
|
||||
@@ -38,7 +38,7 @@ class TestTriggerConditions(unittest.TestCase):
|
||||
first_content = "I saw 100 people at a rock show"
|
||||
timestamp1 = int(time.time())
|
||||
snapshot_id1 = str(uuid.uuid4())
|
||||
watch.save_history_text(contents=first_content,
|
||||
watch.save_history_blob(contents=first_content,
|
||||
timestamp=timestamp1,
|
||||
snapshot_id=snapshot_id1)
|
||||
|
||||
@@ -46,7 +46,7 @@ class TestTriggerConditions(unittest.TestCase):
|
||||
second_content = "I saw 200 people at a rock show"
|
||||
timestamp2 = int(time.time()) + 60
|
||||
snapshot_id2 = str(uuid.uuid4())
|
||||
watch.save_history_text(contents=second_content,
|
||||
watch.save_history_blob(contents=second_content,
|
||||
timestamp=timestamp2,
|
||||
snapshot_id=snapshot_id2)
|
||||
|
||||
|
||||
@@ -18,12 +18,12 @@ class TestDiffBuilder(unittest.TestCase):
|
||||
|
||||
|
||||
# Contents from the browser are always returned from the browser/requests/etc as str, str is basically UTF-16 in python
|
||||
watch.save_history_text(contents="hello world", timestamp=100, snapshot_id=str(uuid_builder.uuid4()))
|
||||
watch.save_history_text(contents="hello world", timestamp=105, snapshot_id=str(uuid_builder.uuid4()))
|
||||
watch.save_history_text(contents="hello world", timestamp=109, snapshot_id=str(uuid_builder.uuid4()))
|
||||
watch.save_history_text(contents="hello world", timestamp=112, snapshot_id=str(uuid_builder.uuid4()))
|
||||
watch.save_history_text(contents="hello world", timestamp=115, snapshot_id=str(uuid_builder.uuid4()))
|
||||
watch.save_history_text(contents="hello world", timestamp=117, snapshot_id=str(uuid_builder.uuid4()))
|
||||
watch.save_history_blob(contents="hello world", timestamp=100, snapshot_id=str(uuid_builder.uuid4()))
|
||||
watch.save_history_blob(contents="hello world", timestamp=105, snapshot_id=str(uuid_builder.uuid4()))
|
||||
watch.save_history_blob(contents="hello world", timestamp=109, snapshot_id=str(uuid_builder.uuid4()))
|
||||
watch.save_history_blob(contents="hello world", timestamp=112, snapshot_id=str(uuid_builder.uuid4()))
|
||||
watch.save_history_blob(contents="hello world", timestamp=115, snapshot_id=str(uuid_builder.uuid4()))
|
||||
watch.save_history_blob(contents="hello world", timestamp=117, snapshot_id=str(uuid_builder.uuid4()))
|
||||
|
||||
p = watch.get_from_version_based_on_last_viewed
|
||||
assert p == "100", "Correct 'last viewed' timestamp was detected"
|
||||
@@ -53,7 +53,7 @@ class TestDiffBuilder(unittest.TestCase):
|
||||
p = watch.get_from_version_based_on_last_viewed
|
||||
assert p == None, "None when no history available"
|
||||
|
||||
watch.save_history_text(contents="hello world", timestamp=100, snapshot_id=str(uuid_builder.uuid4()))
|
||||
watch.save_history_blob(contents="hello world", timestamp=100, snapshot_id=str(uuid_builder.uuid4()))
|
||||
p = watch.get_from_version_based_on_last_viewed
|
||||
assert p == "100", "Correct with only one history snapshot"
|
||||
|
||||
|
||||
Reference in new issue
Block a user