tweaks to saving

This commit is contained in:
dgtlmoon
2022-09-12 12:45:29 +02:00
parent 97c2cd633d
commit 20869a13b3
5 changed files with 14 additions and 5 deletions
@@ -3,7 +3,6 @@ available_fetchers = [('json_html_plaintext', 'JSON/HTML/Text'), ('image', 'Stat
class fetch_processor():
contents = b''
screenshot = None
history_artifact_suffix = 'txt'
"""
base class for all fetch processors
+2 -2
View File
@@ -80,6 +80,8 @@ class perform_site_check(fetch_processor):
update_obj["last_check_status"] = fetcher.get_last_status_code()
self.contents = fetcher.raw_content
image = Image.open(io.BytesIO(fetcher.raw_content))
# @todo different choice?
@@ -97,6 +99,4 @@ class perform_site_check(fetch_processor):
if not watch.get('previous_md5'):
watch['previous_md5'] = fetched_hash
#self.contents = fetcher.screenshot
return changed_detected, update_obj
+10 -1
View File
@@ -146,9 +146,18 @@ class model(dict):
bump = self.history
return self.__newest_history_key
def save_history_artifact(self, contents: bytes, timestamp, suffix='txt'):
def save_history_artifact(self, contents: bytes, timestamp):
import uuid
import logging
import magic
import re
suffix = 'bin'
# detect extension type
mtype = magic.from_buffer(contents, mime=True)
if mtype:
r = re.search(r'image/(\w+)', mtype, re.IGNORECASE)
if r:
suffix = r.group(1)
output_path = "{}/{}".format(self.__datastore_path, self['uuid'])
+1 -1
View File
@@ -270,7 +270,7 @@ class update_worker(threading.Thread):
# For the FIRST time we check a site, or a change detected, save the snapshot.
if changed_detected or not watch['last_checked']:
# A change was detected
watch.save_history_artifact(contents=update_handler.contents, timestamp=str(round(time.time())), suffix=update_handler.history_artifact_suffix)
watch.save_history_artifact(contents=update_handler.contents, timestamp=str(round(time.time())))
self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
+1
View File
@@ -46,3 +46,4 @@ werkzeug ~= 2.0.0
imagehash ~= 4.3.0
pillow
python-magic