Compare commits

..

1 Commits

Author SHA1 Message Date
dgtlmoon 9fcd229435 Remove unused function 2023-02-07 17:25:49 +01:00
15 changed files with 120 additions and 198 deletions
-1
View File
@@ -63,7 +63,6 @@ Requires Playwright to be enabled.
- You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
- Get notified when certain keywords appear in Twitter search results
- Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.
- Get alerts when new job positions are open on Bamboo HR and other job platforms
_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
+2 -3
View File
@@ -36,7 +36,7 @@ from flask import (
from changedetectionio import html_tools
from changedetectionio.api import api_v1
__version__ = '0.40.3'
__version__ = '0.40.2'
datastore = None
@@ -1001,8 +1001,7 @@ def changedetection_app(config=None, datastore_o=None):
os.unlink(previous_backup_filename)
# create a ZipFile object
timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
backupname = "changedetection-backup-{}.zip".format(timestamp)
backupname = "changedetection-backup-{}.zip".format(int(time.time()))
backup_filepath = os.path.join(datastore_o.datastore_path, backupname)
with zipfile.ZipFile(backup_filepath, "w",
+22 -75
View File
@@ -33,7 +33,7 @@ class Watch(Resource):
@auth.check_token
def get(self, uuid):
"""
@api {get} /api/v1/watch/:uuid Get a single watch data
@api {get} /api/v1/watch/:uuid Single watch information
@apiDescription Retrieve watch information and set muted/paused status
@apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -H"x-api-key:813031b16330fe25e3780cf0325daa45"
@@ -70,16 +70,13 @@ class Watch(Resource):
return "OK", 200
# Return without history, get that via another API call
# Properties are not returned as a JSON, so add the required props manually
watch['history_n'] = watch.history_n
watch['last_changed'] = watch.last_changed
return watch
@auth.check_token
def delete(self, uuid):
"""
@api {delete} /api/v1/watch/:uuid Delete a watch and related history
@api {delete} /api/v1/watch/:uuid Delete watch information
@apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45"
@apiParam {uuid} uuid Watch unique ID.
@@ -93,18 +90,21 @@ class Watch(Resource):
self.datastore.delete(uuid)
return 'OK', 204
# Update an existing
@auth.check_token
@expects_json(schema_update_watch)
def put(self, uuid):
"""
@api {put} /api/v1/watch/:uuid Update watch information
@apiExample {curl} Example usage:
Create a watch (POST)
curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
Update (PUT)
curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}'
@apiDescription Updates an existing watch using JSON, accepts the same structure as returned in <a href="#api-Watch-Watch">get single watch information</a>
@apiDescription Updates an existing watch using JSON, accepts the same structure as at https://github.com/dgtlmoon/changedetection.io/blob/fab7d325f764d6912bef671f1d78bf217689c537/changedetectionio/model/Watch.py#L15
@apiParam {uuid} uuid Watch unique ID.
@apiName Update a watch
@apiName Update
@apiGroup Watch
@apiSuccess (200) {String} OK Was updated
@apiSuccess (500) {String} ERR Some other error
@@ -131,21 +131,6 @@ class WatchHistory(Resource):
# Get a list of available history for a watch by UUID
# curl http://localhost:4000/api/v1/watch/<string:uuid>/history
def get(self, uuid):
"""
@api {get} /api/v1/watch/<string:uuid>/history Get a list of all historical snapshots available for a watch
@apiDescription Requires `uuid`, returns list
@apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
{
"1676649279": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/cb7e9be8258368262246910e6a2a4c30.txt",
"1677092785": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/e20db368d6fc633e34f559ff67bb4044.txt",
"1677103794": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/02efdd37dacdae96554a8cc85dc9c945.txt"
}
@apiName Get list of available stored snapshots for watch
@apiGroup Watch History
@apiSuccess (200) {String} OK
@apiSuccess (404) {String} ERR Not found
"""
watch = self.datastore.data['watching'].get(uuid)
if not watch:
abort(404, message='No watch exists with the UUID of {}'.format(uuid))
@@ -157,18 +142,11 @@ class WatchSingleHistory(Resource):
# datastore is a black box dependency
self.datastore = kwargs['datastore']
# Read a given history snapshot and return its content
# <string:timestamp> or "latest"
# curl http://localhost:4000/api/v1/watch/<string:uuid>/history/<int:timestamp>
@auth.check_token
def get(self, uuid, timestamp):
"""
@api {get} /api/v1/watch/<string:uuid>/history/<int:timestamp> Get single snapshot from watch
@apiDescription Requires watch `uuid` and `timestamp`. `timestamp` of "`latest`" for latest available snapshot, or <a href="#api-Watch_History-Get_list_of_available_stored_snapshots_for_watch">use the list returned here</a>
@apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
@apiName Get single snapshot content
@apiGroup Watch History
@apiSuccess (200) {String} OK
@apiSuccess (404) {String} ERR Not found
"""
watch = self.datastore.data['watching'].get(uuid)
if not watch:
abort(404, message='No watch exists with the UUID of {}'.format(uuid))
@@ -179,7 +157,6 @@ class WatchSingleHistory(Resource):
if timestamp == 'latest':
timestamp = list(watch.history.keys())[-1]
# @todo - Check for UTF-8 compatability
with open(watch.history[timestamp], 'r') as f:
content = f.read()
@@ -198,19 +175,21 @@ class CreateWatch(Resource):
@expects_json(schema_create_watch)
def post(self):
"""
@api {post} /api/v1/watch Create a single watch
@apiDescription Requires atleast `url` set, can accept the same structure as <a href="#api-Watch-Watch">get single watch information</a> to create.
@api {post} /api/v1/watch Create a watch
@apiDescription requires `url`, Creates a watch, also accepts accepts the same structure as at https://github.com/dgtlmoon/changedetection.io/blob/fab7d325f764d6912bef671f1d78bf217689c537/changedetectionio/model/Watch.py#L15
@apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
@apiName Create
@apiGroup Watch
@apiGroup CreateWatch
@apiSuccess (200) {String} OK Was created
@apiSuccess (500) {String} ERR Some other error
"""
#
json_data = request.get_json()
url = json_data['url'].strip()
if not validators.url(json_data['url'].strip()):
return "Invalid or unsupported URL", 400
@@ -232,32 +211,17 @@ class CreateWatch(Resource):
@auth.check_token
def get(self):
"""
@api {get} /api/v1/watch List watches
@api {get} /api/v1/watch
@apiDescription Return concise list of available watches and some very basic info
@apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45"
{
"6a4b7d5c-fee4-4616-9f43-4ac97046b595": {
"last_changed": 1677103794,
"last_checked": 1677103794,
"last_error": false,
"title": "",
"url": "http://www.quotationspage.com/random.php"
},
"e6f5fd5c-dbfe-468b-b8f3-f9d6ff5ad69b": {
"last_changed": 0,
"last_checked": 1676662819,
"last_error": false,
"title": "QuickLook",
"url": "https://github.com/QL-Win/QuickLook/tags"
}
}
recheck_all=1 to recheck all
@apiParam {String} [recheck_all] Optional Set to =1 to force recheck of all watches
@apiParam {String} [tag] Optional name of tag to limit results
@apiName ListWatches
@apiGroup Watch Management
@apiSuccess (200) {String} OK JSON dict
@apiGroup CreateWatch
:return:
"""
list = {}
@@ -288,22 +252,6 @@ class SystemInfo(Resource):
@auth.check_token
def get(self):
"""
@api {get} /api/v1/systeminfo Return system info
@apiDescription Return some info about the current system state
@apiExample {curl} Example usage:
curl http://localhost:4000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45"
HTTP/1.0 200
{
'queue_size': 10 ,
'overdue_watches': ["watch-uuid-list"],
'uptime': 38344.55,
'watch_count': 800,
'version': "0.40.1"
}
@apiName Get Info
@apiGroup System Information
"""
import time
overdue_watches = []
@@ -322,11 +270,10 @@ class SystemInfo(Resource):
# Allow 5 minutes of grace time before we decide it's overdue
if time_since_check - (5 * 60) > t:
overdue_watches.append(uuid)
from changedetectionio import __version__ as main_version
return {
'queue_size': self.update_q.qsize(),
'overdue_watches': overdue_watches,
'uptime': round(time.time() - self.datastore.start_time, 2),
'watch_count': len(self.datastore.data.get('watching', {})),
'version': main_version
'watch_count': len(self.datastore.data.get('watching', {}))
}, 200
@@ -31,7 +31,6 @@ browser_step_ui_config = {'Choose one': '0 0',
'Uncheck checkbox': '1 0',
'Wait for seconds': '0 1',
'Wait for text': '0 1',
'Wait for text in element': '1 1',
# 'Press Page Down': '0 0',
# 'Press Page Up': '0 0',
# weird bug, come back to it later
@@ -133,17 +132,6 @@ class steppable_browser_interface():
def action_wait_for_seconds(self, selector, value):
self.page.wait_for_timeout(int(value) * 1000)
def action_wait_for_text(self, selector, value):
import json
v = json.dumps(value)
self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=30000)
def action_wait_for_text_in_element(self, selector, value):
import json
s = json.dumps(selector)
v = json.dumps(value)
self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=30000)
# @todo - in the future make some popout interface to capture what needs to be set
# https://playwright.dev/python/docs/api/class-keyboard
def action_press_enter(self, selector, value):
+8 -13
View File
@@ -153,9 +153,7 @@ class model(dict):
@property
def is_pdf(self):
# content_type field is set in the future
# https://github.com/dgtlmoon/changedetection.io/issues/1392
# Not sure the best logic here
return self.get('url', '').lower().endswith('.pdf') or 'pdf' in self.get('content_type', '').lower()
return '.pdf' in self.get('url', '').lower() or 'pdf' in self.get('content_type', '').lower()
@property
def label(self):
@@ -241,7 +239,7 @@ class model(dict):
# Save some text file to the appropriate path and bump the history
# result_obj from fetch_site_status.run()
def save_history_text(self, contents, timestamp, snapshot_id):
def save_history_text(self, contents, timestamp):
self.ensure_data_dir_exists()
@@ -250,16 +248,13 @@ class model(dict):
if self.__newest_history_key and int(timestamp) == int(self.__newest_history_key):
time.sleep(timestamp - self.__newest_history_key)
snapshot_fname = f"{snapshot_id}.txt"
snapshot_fname = "{}.txt".format(str(uuid.uuid4()))
# Only write if it does not exist, this is so that we dont bother re-saving the same data by checksum under different filenames.
dest = os.path.join(self.watch_data_dir, snapshot_fname)
if not os.path.exists(dest):
# in /diff/ and /preview/ we are going to assume for now that it's UTF-8 when reading
# most sites are utf-8 and some are even broken utf-8
with open(dest, 'wb') as f:
f.write(contents)
f.close()
# in /diff/ and /preview/ we are going to assume for now that it's UTF-8 when reading
# most sites are utf-8 and some are even broken utf-8
with open(os.path.join(self.watch_data_dir, snapshot_fname), 'wb') as f:
f.write(contents)
f.close()
# Append to index
# @todo check last char was \n
+11 -8
View File
@@ -192,24 +192,27 @@ class ChangeDetectionStore:
tags.sort()
return tags
def unlink_history_file(self, path):
try:
unlink(path)
except (FileNotFoundError, IOError):
pass
# Delete a single watch by UUID
def delete(self, uuid):
import pathlib
import shutil
with self.lock:
if uuid == 'all':
self.__data['watching'] = {}
# GitHub #30 also delete history records
for uuid in self.data['watching']:
path = pathlib.Path(os.path.join(self.datastore_path, uuid))
shutil.rmtree(path)
self.needs_write_urgent = True
for path in self.data['watching'][uuid].history.values():
self.unlink_history_file(path)
else:
path = pathlib.Path(os.path.join(self.datastore_path, uuid))
shutil.rmtree(path)
for path in self.data['watching'][uuid].history.values():
self.unlink_history_file(path)
del self.data['watching'][uuid]
self.needs_write_urgent = True
@@ -57,9 +57,9 @@
<th></th>
{% set link_order = "desc" if sort_order else "asc" %}
{% set arrow_span = "" %}
<th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('index', sort='label', order=link_order, tag=active_tag)}}">Website <span class='arrow {{link_order}}'></span></a></th>
<th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order, tag=active_tag)}}">Last Checked <span class='arrow {{link_order}}'></span></a></th>
<th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order, tag=active_tag)}}">Last Changed <span class='arrow {{link_order}}'></span></a></th>
<th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('index', sort='label', order=link_order)}}">Website <span class='arrow {{link_order}}'></span></a></th>
<th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order)}}">Last Checked <span class='arrow {{link_order}}'></span></a></th>
<th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order)}}">Last Changed <span class='arrow {{link_order}}'></span></a></th>
<th></th>
</tr>
</thead>
@@ -117,3 +117,18 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
assert 'Ticket now on sale' in notification
os.unlink("test-datastore/notification.txt")
# Test that if it gets removed, then re-added, we get a notification
# Remove the target and re-add it, we should get a new notification
set_response_without_filter()
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(3)
assert not os.path.isfile("test-datastore/notification.txt")
set_response_with_filter()
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(3)
assert os.path.isfile("test-datastore/notification.txt")
# Also test that the filter was updated after the first one was requested
+22 -18
View File
@@ -169,8 +169,10 @@ class update_worker(threading.Thread):
if uuid in list(self.datastore.data['watching'].keys()):
changed_detected = False
contents = b''
process_changedetection_results = True
screenshot = False
update_obj= {}
xpath_data = False
process_changedetection_results = True
print("> Processing UUID {} Priority {} URL {}".format(uuid, queued_item_data.priority, self.datastore.data['watching'][uuid]['url']))
now = time.time()
@@ -210,7 +212,9 @@ class update_worker(threading.Thread):
if e.page_text:
self.datastore.save_error_text(watch_uuid=uuid, contents=e.page_text)
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
# So that we get a trigger when the content is added again
'previous_md5': ''})
process_changedetection_results = False
except FilterNotFoundInResponse as e:
@@ -218,7 +222,9 @@ class update_worker(threading.Thread):
continue
err_text = "Warning, no filters were found, no change detection ran."
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
# So that we get a trigger when the content is added again
'previous_md5': ''})
# Only when enabled, send the notification
if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
@@ -235,12 +241,11 @@ class update_worker(threading.Thread):
self.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
process_changedetection_results = False
process_changedetection_results = True
except content_fetcher.checksumFromPreviousCheckWasTheSame as e:
# Yes fine, so nothing todo, don't continue to process.
process_changedetection_results = False
changed_detected = False
# Yes fine, so nothing todo
pass
except content_fetcher.BrowserStepsStepTimout as e:
@@ -248,7 +253,9 @@ class update_worker(threading.Thread):
continue
err_text = "Warning, browser step at position {} could not run, target not found, check the watch, add a delay if necessary.".format(e.step_n+1)
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
# So that we get a trigger when the content is added again
'previous_md5': ''})
if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
@@ -264,7 +271,6 @@ class update_worker(threading.Thread):
c = 0
self.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
process_changedetection_results = False
except content_fetcher.EmptyReply as e:
@@ -272,7 +278,6 @@ class update_worker(threading.Thread):
err_text = "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code {}".format(e.status_code)
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
'last_check_status': e.status_code})
process_changedetection_results = False
except content_fetcher.ScreenshotUnavailable as e:
err_text = "Screenshot unavailable, page did not render fully in the expected time - try increasing 'Wait seconds before extracting text'"
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
@@ -284,7 +289,6 @@ class update_worker(threading.Thread):
self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
'last_check_status': e.status_code})
process_changedetection_results = False
except content_fetcher.PageUnloadable as e:
err_text = "Page request from server didnt respond correctly"
if e.message:
@@ -295,7 +299,6 @@ class update_worker(threading.Thread):
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
'last_check_status': e.status_code})
process_changedetection_results = False
except Exception as e:
self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
@@ -315,14 +318,15 @@ class update_worker(threading.Thread):
# Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
if process_changedetection_results:
try:
watch = self.datastore.data['watching'].get(uuid)
self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
watch = self.datastore.data['watching'][uuid]
fname = "" # Saved history text filename
# Also save the snapshot on the first time checked
# For the FIRST time we check a site, or a change detected, save the snapshot.
if changed_detected or not watch['last_checked']:
watch.save_history_text(contents=contents,
timestamp=str(round(time.time())),
snapshot_id=update_obj.get('previous_md5', 'none'))
# A change was detected
watch.save_history_text(contents=contents, timestamp=str(round(time.time())))
self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
# A change was detected
if changed_detected:
File diff suppressed because one or more lines are too long
-1
View File
@@ -49,7 +49,6 @@ input[type="date"] {
src: url('./glyphicons-halflings-regular.eot');
src: url('./glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'),
url('./glyphicons-halflings-regular.woff') format('woff'),
url('./glyphicons-halflings-regular.woff2') format('woff2'),
url('./glyphicons-halflings-regular.ttf') format('truetype'),
url('./glyphicons-halflings-regular.svg#glyphicons-halflingsregular') format('svg');
}
+8 -8
View File
@@ -5,13 +5,13 @@
<meta name="description" content="Manage your changedetection.io watches via API, requires the `x-api-key` header which is found in the settings UI.">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<link href="assets/bootstrap.min.css?v=1677105736053" rel="stylesheet" media="screen">
<link href="assets/prism.css?v=1677105736053" rel="stylesheet" />
<link href="assets/main.css?v=1677105736053" rel="stylesheet" media="screen, print">
<link href="assets/favicon.ico?v=1677105736053" rel="icon" type="image/x-icon">
<link href="assets/apple-touch-icon.png?v=1677105736053" rel="apple-touch-icon" sizes="180x180">
<link href="assets/favicon-32x32.png?v=1677105736053" rel="icon" type="image/png" sizes="32x32">
<link href="assets/favicon-16x16.png?v=1677105736053" rel="icon" type="image/png" sizes="16x16">
<link href="assets/bootstrap.min.css" rel="stylesheet" media="screen">
<link href="assets/prism.css" rel="stylesheet" />
<link href="assets/main.css" rel="stylesheet" media="screen, print">
<link href="assets/favicon.ico" rel="icon" type="image/x-icon">
<link href="assets/apple-touch-icon.png" rel="apple-touch-icon" sizes="180x180">
<link href="assets/favicon-32x32.png" rel="icon" type="image/png" sizes="32x32">
<link href="assets/favicon-16x16.png"rel="icon" type="image/png" sizes="16x16">
</head>
<body class="container-fluid">
@@ -928,6 +928,6 @@
</div>
</div>
<script src="assets/main.bundle.js?v=1677105736053"></script>
<script src="assets/main.bundle.js"></script>
</body>
</html>
+1 -2
View File
@@ -3,6 +3,5 @@
"version": "0.1.0",
"description": "Manage your changedetection.io watches via API, requires the `x-api-key` header which is found in the settings UI.",
"title": "changedetection.io API",
"url" : "",
"sampleUrl" : false
"url" : "https://changedetection.io/docs/api_v1/index.html"
}
+1 -1
View File
@@ -1,5 +1,5 @@
{
"dependencies": {
"apidoc": "^0.54.0"
"apidoc": "^0.53.1"
}
}
+2 -2
View File
@@ -31,7 +31,7 @@ dnspython<2.3.0
# jq not available on Windows so must be installed manually
# Notification library
apprise~=1.3.0
apprise~=1.2.1
# apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
paho-mqtt
@@ -42,7 +42,7 @@ paho-mqtt
cryptography~=3.4
# Used for CSS filtering
beautifulsoup4
bs4
# XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
lxml