diff --git a/backend/backend.py b/backend/backend.py index dc05ed70..3f6b43a9 100644 --- a/backend/backend.py +++ b/backend/backend.py @@ -2,6 +2,14 @@ # @todo logging +# @todo sort by last_changed +# @todo extra options for url like , verify=False etc. +# @todo enable https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl as option? +# @todo maybe a button to reset all 'last-changed'.. so you can see it clearly when something happens since your last visit +# @todo option for interval day/6 hour/etc +# @todo on change detected, config for calling some API +# @todo make tables responsive! +# @todo fetch title into json import json import eventlet @@ -12,6 +20,7 @@ import os import getopt import sys import datetime +import timeago import threading @@ -46,16 +55,23 @@ def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"): return "Checking now.." if watch_obj['last_checked'] == 0: - return 'Never' + return 'Not yet' return datetime.datetime.utcfromtimestamp(int(watch_obj['last_checked'])).strftime(format) +#@app.context_processor +#def timeago(): +# def _timeago(lower_time, now): +# return timeago.format(lower_time, now) +# return dict(timeago=_timeago) -@app.template_filter('format_timestamp') +@app.template_filter('format_timestamp_timeago') def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"): if timestamp == 0: - return 'Never' - return datetime.datetime.utcfromtimestamp(timestamp).strftime(format) + return 'Not yet' + return timeago.format(timestamp, time.time()) + #return timeago.format(timestamp, time.time()) + #return datetime.datetime.utcfromtimestamp(timestamp).strftime(format) @app.route("/", methods=['GET']) @@ -111,7 +127,7 @@ def launch_checks(): global running_update_threads for watch in datastore.data['watching']: - if watch['last_checked'] <= time.time() - 86400: + if watch['last_checked'] <= time.time() - 3*60*60: running_update_threads[watch['uuid']] = fetch_site_status.perform_site_check(uuid=watch['uuid'], datastore=datastore) running_update_threads[watch['uuid']].start() diff --git a/backend/fetch_site_status.py b/backend/fetch_site_status.py index 7dee51b8..b8b7bea7 100644 --- a/backend/fetch_site_status.py +++ b/backend/fetch_site_status.py @@ -45,19 +45,20 @@ class perform_site_check(Thread): import html2text try: - r = requests.get(self.url, headers=headers, timeout=15) - + r = requests.get(self.url, headers=headers, timeout=15, verify=False) stripped_text_from_html = html2text.html2text(r.content.decode('utf-8')) + # Usually from networkIO/requests level except (requests.exceptions.ConnectionError,requests.exceptions.ReadTimeout) as e: self.datastore.update_watch(self.uuid, 'last_error', str(e)) print(str(e)) - # Usually from html2text level except UnicodeDecodeError as e: self.datastore.update_watch(self.uuid, 'last_error', str(e)) print(str(e)) + # figure out how to deal with this cleaner.. + # 'utf-8' codec can't decode byte 0xe9 in position 480: invalid continuation byte else: @@ -72,9 +73,14 @@ class perform_site_check(Thread): fetched_md5 = hashlib.md5(stripped_text_from_html.encode('utf-8')).hexdigest() if self.current_md5 != fetched_md5: + + # Dont confuse people by putting last-changed, when it actually just changed from nothing.. + if self.datastore.get_val(self.uuid, 'previous_md5') is not None: + self.datastore.update_watch(self.uuid, 'last_changed', self.timestamp) + self.datastore.update_watch(self.uuid, 'previous_md5', fetched_md5) self.save_response_output(r.text) - self.datastore.update_watch(self.uuid, 'last_changed', self.timestamp) + self.datastore.update_watch(self.uuid, 'last_checked', int(time.time())) pass diff --git a/backend/static/css/styles.css b/backend/static/css/styles.css index 9fed6971..4e57305e 100644 --- a/backend/static/css/styles.css +++ b/backend/static/css/styles.css @@ -4,9 +4,10 @@ */ body { color: #333; + background: #262626; } - +/* Some styles from https://css-tricks.com/ */ a { text-decoration: none; @@ -18,13 +19,61 @@ a.github-link { section.content { - padding-top: 3em; + padding-top: 5em; + padding-bottom: 5em; flex-direction: column; display: flex; align-items: center; justify-content: center; } +.watch-table { +font-size: 90%; + +} + .watch-table .error { color: #aa0000; +} + +.home-menu { +background: #fff; +padding: 5px; + display: flex; + justify-content: space-between; +} + +.pure-table-even { +/* missing */ +background: #fff; +} +body:after { + content: ""; + background: linear-gradient(130deg,#ff7a18,#af002d 41.07%,#319197 76.05%) +} + +body:after,body:before { + display: block; + height: 600px; + position: absolute; + top: 0; + left: 0; + width: 100%; + z-index: -1; +} + body::after { + opacity: 0.91; + } + body::before { + content: ""; + background-image: + url(/static/images/gradient-border.png); + } +body:before { + background-size: cover +} + +body:after,body:before { + -webkit-clip-path: polygon(100% 0,0 0,0 77.5%,1% 77.4%,2% 77.1%,3% 76.6%,4% 75.9%,5% 75.05%,6% 74.05%,7% 72.95%,8% 71.75%,9% 70.55%,10% 69.3%,11% 68.05%,12% 66.9%,13% 65.8%,14% 64.8%,15% 64%,16% 63.35%,17% 62.85%,18% 62.6%,19% 62.5%,20% 62.65%,21% 63%,22% 63.5%,23% 64.2%,24% 65.1%,25% 66.1%,26% 67.2%,27% 68.4%,28% 69.65%,29% 70.9%,30% 72.15%,31% 73.3%,32% 74.35%,33% 75.3%,34% 76.1%,35% 76.75%,36% 77.2%,37% 77.45%,38% 77.5%,39% 77.3%,40% 76.95%,41% 76.4%,42% 75.65%,43% 74.75%,44% 73.75%,45% 72.6%,46% 71.4%,47% 70.15%,48% 68.9%,49% 67.7%,50% 66.55%,51% 65.5%,52% 64.55%,53% 63.75%,54% 63.15%,55% 62.75%,56% 62.55%,57% 62.5%,58% 62.7%,59% 63.1%,60% 63.7%,61% 64.45%,62% 65.4%,63% 66.45%,64% 67.6%,65% 68.8%,66% 70.05%,67% 71.3%,68% 72.5%,69% 73.6%,70% 74.65%,71% 75.55%,72% 76.35%,73% 76.9%,74% 77.3%,75% 77.5%,76% 77.45%,77% 77.25%,78% 76.8%,79% 76.2%,80% 75.4%,81% 74.45%,82% 73.4%,83% 72.25%,84% 71.05%,85% 69.8%,86% 68.55%,87% 67.35%,88% 66.2%,89% 65.2%,90% 64.3%,91% 63.55%,92% 63%,93% 62.65%,94% 62.5%,95% 62.55%,96% 62.8%,97% 63.3%,98% 63.9%,99% 64.75%,100% 65.7%); + clip-path: polygon(100% 0,0 0,0 77.5%,1% 77.4%,2% 77.1%,3% 76.6%,4% 75.9%,5% 75.05%,6% 74.05%,7% 72.95%,8% 71.75%,9% 70.55%,10% 69.3%,11% 68.05%,12% 66.9%,13% 65.8%,14% 64.8%,15% 64%,16% 63.35%,17% 62.85%,18% 62.6%,19% 62.5%,20% 62.65%,21% 63%,22% 63.5%,23% 64.2%,24% 65.1%,25% 66.1%,26% 67.2%,27% 68.4%,28% 69.65%,29% 70.9%,30% 72.15%,31% 73.3%,32% 74.35%,33% 75.3%,34% 76.1%,35% 76.75%,36% 77.2%,37% 77.45%,38% 77.5%,39% 77.3%,40% 76.95%,41% 76.4%,42% 75.65%,43% 74.75%,44% 73.75%,45% 72.6%,46% 71.4%,47% 70.15%,48% 68.9%,49% 67.7%,50% 66.55%,51% 65.5%,52% 64.55%,53% 63.75%,54% 63.15%,55% 62.75%,56% 62.55%,57% 62.5%,58% 62.7%,59% 63.1%,60% 63.7%,61% 64.45%,62% 65.4%,63% 66.45%,64% 67.6%,65% 68.8%,66% 70.05%,67% 71.3%,68% 72.5%,69% 73.6%,70% 74.65%,71% 75.55%,72% 76.35%,73% 76.9%,74% 77.3%,75% 77.5%,76% 77.45%,77% 77.25%,78% 76.8%,79% 76.2%,80% 75.4%,81% 74.45%,82% 73.4%,83% 72.25%,84% 71.05%,85% 69.8%,86% 68.55%,87% 67.35%,88% 66.2%,89% 65.2%,90% 64.3%,91% 63.55%,92% 63%,93% 62.65%,94% 62.5%,95% 62.55%,96% 62.8%,97% 63.3%,98% 63.9%,99% 64.75%,100% 65.7%) } \ No newline at end of file diff --git a/backend/store.py b/backend/store.py index 9eefabb8..5865143d 100644 --- a/backend/store.py +++ b/backend/store.py @@ -13,9 +13,9 @@ class ChangeDetectionStore: try: with open('/datastore/url-watches.json') as json_file: self.data = json.load(json_file) + self.data['watching'].reverse() for p in self.data['watching']: - print('url: ' + p['url']) - print('') + print("Watching:", p['url']) # First time ran, doesnt exist. except (FileNotFoundError, json.decoder.JSONDecodeError): @@ -51,6 +51,15 @@ class ChangeDetectionStore: self.sync_to_json() + def url_exists(self, url): + + # Probably their should be dict... + for watch in self.data['watching']: + if watch['url'] == url: + return True + + return False + def get_val(self, uuid, val): # Probably their should be dict... for watch in self.data['watching']: @@ -78,6 +87,6 @@ class ChangeDetectionStore: def sync_to_json(self): with open('/datastore/url-watches.json', 'w') as json_file: - json.dump(self.data, json_file) + json.dump(self.data, json_file, indent=4) # body of the constructor \ No newline at end of file diff --git a/backend/templates/base.html b/backend/templates/base.html index f4604023..c540aa86 100644 --- a/backend/templates/base.html +++ b/backend/templates/base.html @@ -12,10 +12,10 @@
- ChangeDetection + ChangeDetection.io
diff --git a/backend/templates/watch-overview.html b/backend/templates/watch-overview.html index da66c874..c706de78 100644 --- a/backend/templates/watch-overview.html +++ b/backend/templates/watch-overview.html @@ -15,7 +15,7 @@ - + Note: Times are in UTC for now - todo - JS front end format
 @@ -23,7 +23,7 @@ - + @@ -36,7 +36,7 @@ - + {% endfor %}
URL Last Checked Last Changedop
{{watch|format_last_checked_time}} {% if watch.last_error is defined and watch.last_error != False %} !! {% endif %} {{watch.last_changed|format_timestamp}}{{watch.last_changed|format_timestamp_timeago}} Recheck