lee/changedetection.io
1
0
Fork 0
mirror of https://github.com/dgtlmoon/changedetection.io.git synced 2025-12-11 18:45:34 +00:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

Handle errors better, use the plaintext output

Browse Source
This commit is contained in:
Leigh Morresi
2021-01-27 15:56:59 +01:00
parent 2f018ac04c
commit 646a54945a
6 changed files with 68 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
backend/fetch_site_status.py
View File

@@ -1,21 +1,21 @@
from threading import Thread
import time
import requests
import hashlib
# Hmm Polymorphism datastore, thread, etc
class perform_site_check(Thread):
def __init__(self, *args, uuid=False, datastore, **kwargs):
super().__init__(*args, **kwargs)
self.timestamp = int(time.time()) # used for storage etc too
self.timestamp = int(time.time()) # used for storage etc too
self.uuid = uuid
self.datastore = datastore
self.url = datastore.get_val(uuid, 'url')
self.current_md5 = datastore.get_val(uuid, 'previous_md5')
def save_firefox_screenshot(self, uuid, output):
#@todo call selenium or whatever
# @todo call selenium or whatever
return
def save_response_output(self, output):
@@ -31,19 +31,45 @@ class perform_site_check(Thread):
f.write(output)
f.close()
def run(self):
try:
r = requests.get(self.url)
except requests.exceptions.ConnectionError as e:
self.datastore.update_watch(self.uuid, 'last_error', str(e))
print (str(e))
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8,cs;q=0.7'
}
print("Checking", self.url)
import html2text
try:
r = requests.get(self.url, headers=headers, timeout=15)
stripped_text_from_html = html2text.html2text(r.content.decode('utf-8'))
# Usually from networkIO/requests level
except (requests.exceptions.ConnectionError,requests.exceptions.ReadTimeout) as e:
self.datastore.update_watch(self.uuid, 'last_error', str(e))
print(str(e))
# Usually from html2text level
except UnicodeDecodeError as e:
self.datastore.update_watch(self.uuid, 'last_error', str(e))
print(str(e))
else:
# We rely on the actual text in the html output.. many sites have random script vars etc
self.datastore.update_watch(self.uuid, 'last_error', False)
self.datastore.update_watch(self.uuid, 'last_check_status', r.status_code)
fetched_md5=hashlib.md5(r.text.encode('utf-8')).hexdigest()
fetched_md5 = hashlib.md5(stripped_text_from_html.encode('utf-8')).hexdigest()
if self.current_md5 != fetched_md5:
self.datastore.update_watch(self.uuid, 'previous_md5', fetched_md5)