lee/changedetection.io
1
0
Fork 0
mirror of https://github.com/dgtlmoon/changedetection.io.git synced 2025-12-14 03:56:08 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Feature - priority queue - edited and added watches should get checked before automatically queued watches (#799)

Browse Source
This commit is contained in:
dgtlmoon
2022-07-31 15:35:35 +02:00
committed by GitHub
parent 8fef3ff4ab
commit a1c3107cd6
3 changed files with 19 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
changedetectionio/__init__.py
View File

@@ -54,7 +54,7 @@ ticker_thread = None
extra_stylesheets = [] extra_stylesheets = []
update_q = queue.Queue() update_q = queue.PriorityQueue()
notification_q = queue.Queue() notification_q = queue.Queue()
@@ -406,7 +406,6 @@ def changedetection_app(config=None, datastore_o=None):
existing_tags = datastore.get_all_tags() existing_tags = datastore.get_all_tags()
form = forms.quickWatchForm(request.form) form = forms.quickWatchForm(request.form)
output = render_template("watch-overview.html", output = render_template("watch-overview.html",
form=form, form=form,
watches=sorted_watches, watches=sorted_watches,
@@ -417,7 +416,7 @@ def changedetection_app(config=None, datastore_o=None):
# Don't link to hosting when we're on the hosting environment # Don't link to hosting when we're on the hosting environment
hosted_sticky=os.getenv("SALTED_PASS", False) == False, hosted_sticky=os.getenv("SALTED_PASS", False) == False,
guid=datastore.data['app_guid'], guid=datastore.data['app_guid'],
queued_uuids=update_q.queue) queued_uuids=[uuid for p,uuid in update_q.queue])
if session.get('share-link'): if session.get('share-link'):
@@ -631,8 +630,8 @@ def changedetection_app(config=None, datastore_o=None):
# But in the case something is added we should save straight away # But in the case something is added we should save straight away
datastore.needs_write_urgent = True datastore.needs_write_urgent = True
# Queue the watch for immediate recheck # Queue the watch for immediate recheck, with a higher priority
update_q.put(uuid) update_q.put((1, uuid))
# Diff page [edit] link should go back to diff page # Diff page [edit] link should go back to diff page
if request.args.get("next") and request.args.get("next") == 'diff' and not form.save_and_preview_button.data: if request.args.get("next") and request.args.get("next") == 'diff' and not form.save_and_preview_button.data:
@@ -746,7 +745,7 @@ def changedetection_app(config=None, datastore_o=None):
importer = import_url_list() importer = import_url_list()
importer.run(data=request.values.get('urls'), flash=flash, datastore=datastore) importer.run(data=request.values.get('urls'), flash=flash, datastore=datastore)
for uuid in importer.new_uuids: for uuid in importer.new_uuids:
update_q.put(uuid) update_q.put((1, uuid))
if len(importer.remaining_data) == 0: if len(importer.remaining_data) == 0:
return redirect(url_for('index')) return redirect(url_for('index'))
@@ -759,7 +758,7 @@ def changedetection_app(config=None, datastore_o=None):
d_importer = import_distill_io_json() d_importer = import_distill_io_json()
d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore) d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore)
for uuid in d_importer.new_uuids: for uuid in d_importer.new_uuids:
update_q.put(uuid) update_q.put((1, uuid))
@@ -1088,7 +1087,7 @@ def changedetection_app(config=None, datastore_o=None):
if not add_paused and new_uuid: if not add_paused and new_uuid:
# Straight into the queue. # Straight into the queue.
update_q.put(new_uuid) update_q.put((1, new_uuid))
flash("Watch added.") flash("Watch added.")
if add_paused: if add_paused:
@@ -1125,7 +1124,7 @@ def changedetection_app(config=None, datastore_o=None):
uuid = list(datastore.data['watching'].keys()).pop() uuid = list(datastore.data['watching'].keys()).pop()
new_uuid = datastore.clone(uuid) new_uuid = datastore.clone(uuid)
update_q.put(new_uuid) update_q.put((5, new_uuid))
flash('Cloned.') flash('Cloned.')
return redirect(url_for('index')) return redirect(url_for('index'))
@@ -1146,7 +1145,7 @@ def changedetection_app(config=None, datastore_o=None):
if uuid: if uuid:
if uuid not in running_uuids: if uuid not in running_uuids:
update_q.put(uuid) update_q.put((1, uuid))
i = 1 i = 1
elif tag != None: elif tag != None:
@@ -1154,7 +1153,7 @@ def changedetection_app(config=None, datastore_o=None):
for watch_uuid, watch in datastore.data['watching'].items(): for watch_uuid, watch in datastore.data['watching'].items():
if (tag != None and tag in watch['tag']): if (tag != None and tag in watch['tag']):
if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']: if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
update_q.put(watch_uuid) update_q.put((1, watch_uuid))
i += 1 i += 1
else: else:
@@ -1162,7 +1161,7 @@ def changedetection_app(config=None, datastore_o=None):
for watch_uuid, watch in datastore.data['watching'].items(): for watch_uuid, watch in datastore.data['watching'].items():
if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']: if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
update_q.put(watch_uuid) update_q.put((1, watch_uuid))
i += 1 i += 1
flash("{} watches are queued for rechecking.".format(i)) flash("{} watches are queued for rechecking.".format(i))
return redirect(url_for('index', tag=tag)) return redirect(url_for('index', tag=tag))
@@ -1366,14 +1365,14 @@ def ticker_thread_check_time_launch_checks():
seconds_since_last_recheck = now - watch['last_checked'] seconds_since_last_recheck = now - watch['last_checked']
if seconds_since_last_recheck >= (threshold + watch.jitter_seconds) and seconds_since_last_recheck >= recheck_time_minimum_seconds: if seconds_since_last_recheck >= (threshold + watch.jitter_seconds) and seconds_since_last_recheck >= recheck_time_minimum_seconds:
if not uuid in running_uuids and uuid not in update_q.queue: if not uuid in running_uuids and uuid not in [q_uuid for p,q_uuid in update_q.queue]:
print("Queued watch UUID {} last checked at {} queued at {:0.2f} jitter {:0.2f}s, {:0.2f}s since last checked".format(uuid, print("> Queued watch UUID {} last checked at {} queued at {:0.2f} jitter {:0.2f}s, {:0.2f}s since last checked".format(uuid,
watch['last_checked'], watch['last_checked'],
now, now,
watch.jitter_seconds, watch.jitter_seconds,
now - watch['last_checked'])) now - watch['last_checked']))
# Into the queue with you # Into the queue with you
update_q.put(uuid) update_q.put((5, uuid))
# Reset for next time # Reset for next time
watch.jitter_seconds = 0 watch.jitter_seconds = 0

6
changedetectionio/api/api_v1.py
View File

@@ -24,7 +24,7 @@ class Watch(Resource):
abort(404, message='No watch exists with the UUID of {}'.format(uuid)) abort(404, message='No watch exists with the UUID of {}'.format(uuid))
if request.args.get('recheck'): if request.args.get('recheck'):
self.update_q.put(uuid) self.update_q.put((1, uuid))
return "OK", 200 return "OK", 200
# Return without history, get that via another API call # Return without history, get that via another API call
@@ -100,7 +100,7 @@ class CreateWatch(Resource):
extras = {'title': json_data['title'].strip()} if json_data.get('title') else {} extras = {'title': json_data['title'].strip()} if json_data.get('title') else {}
new_uuid = self.datastore.add_watch(url=json_data['url'].strip(), tag=tag, extras=extras) new_uuid = self.datastore.add_watch(url=json_data['url'].strip(), tag=tag, extras=extras)
self.update_q.put(new_uuid) self.update_q.put((1, new_uuid))
return {'uuid': new_uuid}, 201 return {'uuid': new_uuid}, 201
# Return concise list of available watches and some very basic info # Return concise list of available watches and some very basic info
@@ -118,7 +118,7 @@ class CreateWatch(Resource):
if request.args.get('recheck_all'): if request.args.get('recheck_all'):
for uuid in self.datastore.data['watching'].keys(): for uuid in self.datastore.data['watching'].keys():
self.update_q.put(uuid) self.update_q.put((1, uuid))
return {'status': "OK"}, 200 return {'status': "OK"}, 200
return list, 200 return list, 200

5
changedetectionio/update_worker.py
View File

@@ -115,7 +115,7 @@ class update_worker(threading.Thread):
while not self.app.config.exit.is_set(): while not self.app.config.exit.is_set():
try: try:
uuid = self.q.get(block=False) priority, uuid = self.q.get(block=False)
except queue.Empty: except queue.Empty:
pass pass
@@ -123,14 +123,13 @@ class update_worker(threading.Thread):
self.current_uuid = uuid self.current_uuid = uuid
if uuid in list(self.datastore.data['watching'].keys()): if uuid in list(self.datastore.data['watching'].keys()):
changed_detected = False changed_detected = False
contents = b'' contents = b''
screenshot = False screenshot = False
update_obj= {} update_obj= {}
xpath_data = False xpath_data = False
process_changedetection_results = True process_changedetection_results = True
print("> Processing UUID {} Priority {} URL {}".format(uuid, priority, self.datastore.data['watching'][uuid]['url']))
now = time.time() now = time.time()
try: try: