From fff32cef0d76acf5e9f7f08c427e6b7bfcb4594a Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 11 Oct 2022 14:40:36 +0200 Subject: [PATCH 01/25] Adding test - Test the 'execute JS before changedetection' (#1006) --- .../tests/visualselector/test_fetch_data.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/changedetectionio/tests/visualselector/test_fetch_data.py b/changedetectionio/tests/visualselector/test_fetch_data.py index 04cd4644a..17dcfd9f5 100644 --- a/changedetectionio/tests/visualselector/test_fetch_data.py +++ b/changedetectionio/tests/visualselector/test_fetch_data.py @@ -13,9 +13,9 @@ def test_visual_selector_content_ready(client, live_server): live_server_setup(live_server) time.sleep(1) - # Add our URL to the import page, maybe better to use something we control? - # We use an external URL because the docker container is too difficult to setup to connect back to the pytest socket - test_url = 'https://news.ycombinator.com' + # Add our URL to the import page, because the docker container (playwright/selenium) wont be able to connect to our usual test url + test_url = "https://changedetection.io/ci-test/test-runjs.html" + res = client.post( url_for("form_quick_watch_add"), data={"url": test_url, "tag": '', 'edit_and_watch_submit_button': 'Edit > Watch'}, @@ -25,13 +25,27 @@ def test_visual_selector_content_ready(client, live_server): res = client.post( url_for("edit_page", uuid="first", unpause_on_save=1), - data={"css_filter": ".does-not-exist", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_webdriver"}, + data={ + "url": test_url, + "tag": "", + "headers": "", + 'fetch_backend': "html_webdriver", + 'webdriver_js_execute_code': 'document.querySelector("button[name=test-button]").click();' + }, follow_redirects=True ) assert b"unpaused" in res.data time.sleep(1) wait_for_all_checks(client) uuid = extract_UUID_from_client(client) + + # Check the JS execute code before extract worked + res = client.get( + url_for("preview_page", uuid="first"), + follow_redirects=True + ) + assert b'I smell JavaScript' in res.data + assert os.path.isfile(os.path.join('test-datastore', uuid, 'last-screenshot.png')), "last-screenshot.png should exist" assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.json')), "xpath elements.json data should exist" From 32ea1a8721e4ddbd69967a7e4753448f1e2ea6ad Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 12 Oct 2022 09:53:16 +0200 Subject: [PATCH 02/25] Windows - JQ - Make library optional so it doesnt break Windows pip installs (#1009) --- Dockerfile | 5 ++++ README.md | 8 +++-- changedetectionio/__init__.py | 21 ++++++++----- changedetectionio/forms.py | 10 +++++-- changedetectionio/html_tools.py | 18 +++++++---- changedetectionio/run_all_tests.sh | 7 +++++ changedetectionio/templates/edit.html | 8 +++-- .../tests/test_jsonpath_jq_selector.py | 30 ++++++++++++------- requirements.txt | 3 +- 9 files changed, 80 insertions(+), 30 deletions(-) diff --git a/Dockerfile b/Dockerfile index 24d3490e1..d422918e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,11 @@ RUN pip install --target=/dependencies -r /requirements.txt RUN pip install --target=/dependencies playwright~=1.26 \ || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled." + +RUN pip install --target=/dependencies jq~=1.3 \ + || echo "WARN: Failed to install JQ. The application can still run, but the Jq: filter option will be disabled." + + # Final image stage FROM python:3.8-slim diff --git a/README.md b/README.md index 797f8c56b..03b124633 100644 --- a/README.md +++ b/README.md @@ -121,8 +121,8 @@ See the wiki for more information https://github.com/dgtlmoon/changedetection.io ## Filters -XPath, JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. +XPath, JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. (We support LXML `re:test`, `re:math` and `re:replace`.) ## Notifications @@ -163,7 +163,11 @@ This will re-parse the JSON and apply formatting to the text, making it super ea For more complex parsing, filtering, and modifying of JSON data, jq is recommended due to the built-in operators and functions. Refer to the [documentation](https://stedolan.github.io/jq/manual/) for more information on jq. -The example below adds the price in dollars to each item in the JSON data, and then filters to only show items that are greater than 10. +Notes: +- `jq` must be added manually separately from the installation of changedetection.io (simply run `pip3 install jq`) +- `jq` is not available on Windows or must be manually compiled (No "wheel" package available on pypi) + +- The example below adds the price in dollars to each item in the JSON data, and then filters to only show items that are greater than 10. #### Sample input data from API ``` diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 8f6d5a557..e766f78b0 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -636,20 +636,27 @@ def changedetection_app(config=None, datastore_o=None): # Only works reliably with Playwright visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and default['fetch_backend'] == 'html_webdriver' + # JQ is difficult to install on windows and must be manually added (outside requirements.txt) + jq_support = True + try: + import jq + except ModuleNotFoundError: + jq_support = False output = render_template("edit.html", - uuid=uuid, - watch=datastore.data['watching'][uuid], - form=form, - has_empty_checktime=using_default_check_time, - has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False, - using_global_webdriver_wait=default['webdriver_delay'] is None, current_base_url=datastore.data['settings']['application']['base_url'], emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), + form=form, + has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False, + has_empty_checktime=using_default_check_time, + jq_support=jq_support, + playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False), settings_application=datastore.data['settings']['application'], + using_global_webdriver_wait=default['webdriver_delay'] is None, + uuid=uuid, visualselector_data_is_ready=visualselector_data_is_ready, visualselector_enabled=visualselector_enabled, - playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False) + watch=datastore.data['watching'][uuid], ) return output diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 7fa17f90a..51e02884d 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -303,12 +303,16 @@ class ValidateCSSJSONXPATHInput(object): # Re #265 - maybe in the future fetch the page and offer a # warning/notice that its possible the rule doesnt yet match anything? - - if 'jq:' in line: if not self.allow_json: raise ValidationError("jq not permitted in this field!") - import jq + if 'jq:' in line: + try: + import jq + except ModuleNotFoundError: + # `jq` requires full compilation in windows and so isn't generally available + raise ValidationError("jq not support not found") + input = line.replace('jq:', '') try: diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index 6cc8e20ae..167d0f77c 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -1,12 +1,11 @@ -import json -from typing import List from bs4 import BeautifulSoup -from jsonpath_ng.ext import parse -import jq -import re from inscriptis import get_text from inscriptis.model.config import ParserConfig +from jsonpath_ng.ext import parse +from typing import List +import json +import re class FilterNotFoundInResponse(ValueError): def __init__(self, msg): @@ -85,9 +84,18 @@ def _parse_json(json_data, json_filter): jsonpath_expression = parse(json_filter.replace('json:', '')) match = jsonpath_expression.find(json_data) return _get_stripped_text_from_json_match(match) + if 'jq:' in json_filter: + + try: + import jq + except ModuleNotFoundError: + # `jq` requires full compilation in windows and so isn't generally available + raise Exception("jq not support not found") + jq_expression = jq.compile(json_filter.replace('jq:', '')) match = jq_expression.input(json_data).all() + return _get_stripped_text_from_json_match(match) def _get_stripped_text_from_json_match(match): diff --git a/changedetectionio/run_all_tests.sh b/changedetectionio/run_all_tests.sh index e4ea3bac5..28dd85c67 100755 --- a/changedetectionio/run_all_tests.sh +++ b/changedetectionio/run_all_tests.sh @@ -23,6 +23,13 @@ export BASE_URL="https://really-unique-domain.io" pytest tests/test_notification.py +## JQ + JSON: filter test +# jq is not available on windows and we should just test it when the package is installed +# this will re-test with jq support +pip3 install jq~=1.3 +pytest tests/test_jsonpath_jq_selector.py + + # Now for the selenium and playwright/browserless fetchers # Note - this is not UI functional tests - just checking that each one can fetch the content diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 907894e18..59d95317d 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -184,10 +184,14 @@ User-Agent: wonderbra 1.0") }}
  • CSS - Limit text to this CSS rule, only text matching this CSS rule is included.
    • -
  • JSON - Limit text to this JSON rule, using either JSONPath or jq. +
  • JSON - Limit text to this JSON rule, using either JSONPath or jq (if installed).
    • JSONPath: Prefix with json:, use json:$ to force re-formatting if required, test your JSONPath here.
      • + {% if jq_support %}
    • jq: Prefix with jq: and test your jq here. Using jq allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation here.
      • + {% else %} +
    • jq support not installed
      • + {% endif %}
  • XPath - Limit text to this XPath rule, simply start with a forward-slash, @@ -198,7 +202,7 @@ User-Agent: wonderbra 1.0") }}
- Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath, or jq selector rules before filing an issue on GitHub! here for more CSS selector help.
 diff --git a/changedetectionio/tests/test_jsonpath_jq_selector.py b/changedetectionio/tests/test_jsonpath_jq_selector.py index d00821229..f6da84db3 100644 --- a/changedetectionio/tests/test_jsonpath_jq_selector.py +++ b/changedetectionio/tests/test_jsonpath_jq_selector.py @@ -5,7 +5,12 @@ import time from flask import url_for, escape from . util import live_server_setup import pytest +jq_support = True +try: + import jq +except ModuleNotFoundError: + jq_support = False def test_setup(live_server): live_server_setup(live_server) @@ -40,22 +45,24 @@ and it can also be repeated assert text == "23.5" # also check for jq - text = html_tools.extract_json_as_string(content, "jq:.offers.price") - assert text == "23.5" + if jq_support: + text = html_tools.extract_json_as_string(content, "jq:.offers.price") + assert text == "23.5" + + text = html_tools.extract_json_as_string('{"id":5}', "jq:.id") + assert text == "5" text = html_tools.extract_json_as_string('{"id":5}', "json:$.id") assert text == "5" - text = html_tools.extract_json_as_string('{"id":5}', "jq:.id") - assert text == "5" - # When nothing at all is found, it should throw JSONNOTFound # Which is caught and shown to the user in the watch-overview table with pytest.raises(html_tools.JSONNotFound) as e_info: html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "json:$.id") - with pytest.raises(html_tools.JSONNotFound) as e_info: - html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jq:.id") + if jq_support: + with pytest.raises(html_tools.JSONNotFound) as e_info: + html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jq:.id") def set_original_ext_response(): data = """ @@ -271,7 +278,8 @@ def test_check_jsonpath_filter(client, live_server): check_json_filter('json:boss.name', client, live_server) def test_check_jq_filter(client, live_server): - check_json_filter('jq:.boss.name', client, live_server) + if jq_support: + check_json_filter('jq:.boss.name', client, live_server) def check_json_filter_bool_val(json_filter, client, live_server): set_original_response() @@ -329,7 +337,8 @@ def test_check_jsonpath_filter_bool_val(client, live_server): check_json_filter_bool_val("json:$['available']", client, live_server) def test_check_jq_filter_bool_val(client, live_server): - check_json_filter_bool_val("jq:.available", client, live_server) + if jq_support: + check_json_filter_bool_val("jq:.available", client, live_server) # Re #265 - Extended JSON selector test # Stuff to consider here @@ -408,4 +417,5 @@ def test_check_jsonpath_ext_filter(client, live_server): check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server) def test_check_jq_ext_filter(client, live_server): - check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server) \ No newline at end of file + if jq_support: + check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 68aabe9ac..bffc2a7f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,8 @@ chardet > 2.3.0 wtforms ~= 3.0 jsonpath-ng ~= 1.5.3 -jq ~= 1.3.0 + +# jq not available on Windows so must be installed manually # Notification library apprise ~= 1.1.0 From 1b077abd93e7ea9d6ecb7c6c868d9c80d2eed2c9 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 12 Oct 2022 09:53:59 +0200 Subject: [PATCH 03/25] 0.39.20.2 --- changedetectionio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index e766f78b0..c8d8c52f4 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -33,7 +33,7 @@ from flask_wtf import CSRFProtect from changedetectionio import html_tools from changedetectionio.api import api_v1 -__version__ = '0.39.20.1' +__version__ = '0.39.20.2' datastore = None From 8d5b0b5576291185dba8f8cd3ab8551362a714f0 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 12 Oct 2022 10:51:39 +0200 Subject: [PATCH 04/25] Update README.md --- README.md | 44 ++++---------------------------------------- 1 file changed, 4 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 03b124633..0f1678289 100644 --- a/README.md +++ b/README.md @@ -161,50 +161,14 @@ This will re-parse the JSON and apply formatting to the text, making it super ea ### JSONPath or jq? -For more complex parsing, filtering, and modifying of JSON data, jq is recommended due to the built-in operators and functions. Refer to the [documentation](https://stedolan.github.io/jq/manual/) for more information on jq. +For more complex parsing, filtering, and modifying of JSON data, jq is recommended due to the built-in operators and functions. Refer to the [documentation](https://stedolan.github.io/jq/manual/) for more specifc information on jq. -Notes: -- `jq` must be added manually separately from the installation of changedetection.io (simply run `pip3 install jq`) -- `jq` is not available on Windows or must be manually compiled (No "wheel" package available on pypi) +One big advantage of `jq` is that you can use logic in your JSON filter, such as filters to only show items that have a value greater than/less than etc. -- The example below adds the price in dollars to each item in the JSON data, and then filters to only show items that are greater than 10. +See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/JSON-Selector-Filter-help for more information and examples -#### Sample input data from API -``` -{ - "items": [ - { - "name": "Product A", - "priceInCents": 2500 - }, - { - "name": "Product B", - "priceInCents": 500 - }, - { - "name": "Product C", - "priceInCents": 2000 - } - ] -} -``` +Note: `jq` library must be added separately (`pip3 install jq`) -#### Sample jq -`jq:.items[] | . + { "priceInDollars": (.priceInCents / 100) } | select(.priceInDollars > 10)` - -#### Sample output data -``` -{ - "name": "Product A", - "priceInCents": 2500, - "priceInDollars": 25 -} -{ - "name": "Product C", - "priceInCents": 2000, - "priceInDollars": 20 -} -``` ### Parse JSON embedded in HTML! From 63095f70eaf8eb601493bcfb9331044a0d3d82b9 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 17 Oct 2022 17:13:15 +0200 Subject: [PATCH 05/25] Also include tests in pip build --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 3f41f9061..4b3eb3ad8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ recursive-include changedetectionio/api * recursive-include changedetectionio/templates * recursive-include changedetectionio/static * recursive-include changedetectionio/model * +recursive-include changedetectionio/tests * include changedetection.py global-exclude *.pyc global-exclude node_modules From 85897e0bf9315d4a5af2d5a4aa17deec4df33821 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 17 Oct 2022 17:40:28 +0200 Subject: [PATCH 06/25] Windows - diff file handling improvements (#1031) --- changedetectionio/__init__.py | 10 ++++++---- changedetectionio/model/Watch.py | 11 ++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index c8d8c52f4..096070efb 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -816,9 +816,11 @@ def changedetection_app(config=None, datastore_o=None): newest_file = history[dates[-1]] + # Read as binary and force decode as UTF-8 + # Windows may fail decode in python if we just use 'r' mode (chardet decode exception) try: - with open(newest_file, 'r') as f: - newest_version_file_contents = f.read() + with open(newest_file, 'rb') as f: + newest_version_file_contents = f.read().decode('utf-8') except Exception as e: newest_version_file_contents = "Unable to read {}.\n".format(newest_file) @@ -830,8 +832,8 @@ def changedetection_app(config=None, datastore_o=None): previous_file = history[dates[-2]] try: - with open(previous_file, 'r') as f: - previous_version_file_contents = f.read() + with open(previous_file, 'rb') as f: + previous_version_file_contents = f.read().decode('utf-8') except Exception as e: previous_version_file_contents = "Unable to read {}.\n".format(previous_file) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index b7aaca864..9a87ad719 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -151,28 +151,29 @@ class model(dict): import uuid import logging - output_path = "{}/{}".format(self.__datastore_path, self['uuid']) + output_path = os.path.join(self.__datastore_path, self['uuid']) self.ensure_data_dir_exists() + snapshot_fname = os.path.join(output_path, str(uuid.uuid4())) - snapshot_fname = "{}/{}.stripped.txt".format(output_path, uuid.uuid4()) logging.debug("Saving history text {}".format(snapshot_fname)) + # in /diff/ we are going to assume for now that it's UTF-8 when reading with open(snapshot_fname, 'wb') as f: f.write(contents) f.close() # Append to index # @todo check last char was \n - index_fname = "{}/history.txt".format(output_path) + index_fname = os.path.join(output_path, "history.txt") with open(index_fname, 'a') as f: f.write("{},{}\n".format(timestamp, snapshot_fname)) f.close() self.__newest_history_key = timestamp - self.__history_n+=1 + self.__history_n += 1 - #@todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status + # @todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status return snapshot_fname @property From 957a3c1c16baca4f1ee140986ec9e28bfe773a8e Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 17 Oct 2022 17:43:35 +0200 Subject: [PATCH 07/25] 0.39.20.3 --- changedetectionio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 096070efb..c745dd3ec 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -33,7 +33,7 @@ from flask_wtf import CSRFProtect from changedetectionio import html_tools from changedetectionio.api import api_v1 -__version__ = '0.39.20.2' +__version__ = '0.39.20.3' datastore = None From 4be0260381e078052cdd213ab9a6779a7f4c681a Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 17 Oct 2022 18:36:22 +0200 Subject: [PATCH 08/25] Better cross platform file handling in diff and preview (#1034) --- changedetectionio/__init__.py | 10 +++++----- changedetectionio/model/Watch.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index c745dd3ec..19873cceb 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -819,8 +819,8 @@ def changedetection_app(config=None, datastore_o=None): # Read as binary and force decode as UTF-8 # Windows may fail decode in python if we just use 'r' mode (chardet decode exception) try: - with open(newest_file, 'rb') as f: - newest_version_file_contents = f.read().decode('utf-8') + with open(newest_file, 'r', encoding='utf-8', errors='ignore') as f: + newest_version_file_contents = f.read() except Exception as e: newest_version_file_contents = "Unable to read {}.\n".format(newest_file) @@ -832,8 +832,8 @@ def changedetection_app(config=None, datastore_o=None): previous_file = history[dates[-2]] try: - with open(previous_file, 'rb') as f: - previous_version_file_contents = f.read().decode('utf-8') + with open(previous_file, 'r', encoding='utf-8', errors='ignore') as f: + previous_version_file_contents = f.read() except Exception as e: previous_version_file_contents = "Unable to read {}.\n".format(previous_file) @@ -909,7 +909,7 @@ def changedetection_app(config=None, datastore_o=None): timestamp = list(watch.history.keys())[-1] filename = watch.history[timestamp] try: - with open(filename, 'r') as f: + with open(filename, 'r', encoding='utf-8', errors='ignore') as f: tmp = f.readlines() # Get what needs to be highlighted diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 9a87ad719..566eb88e8 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -158,7 +158,8 @@ class model(dict): logging.debug("Saving history text {}".format(snapshot_fname)) - # in /diff/ we are going to assume for now that it's UTF-8 when reading + # in /diff/ and /preview/ we are going to assume for now that it's UTF-8 when reading + # most sites are utf-8 and some are even broken utf-8 with open(snapshot_fname, 'wb') as f: f.write(contents) f.close() From 4cbcc594615a2db474ecad03f211ee613926557d Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 17 Oct 2022 18:36:47 +0200 Subject: [PATCH 09/25] 0.39.20.4 --- changedetectionio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 19873cceb..c6f95f1ea 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -33,7 +33,7 @@ from flask_wtf import CSRFProtect from changedetectionio import html_tools from changedetectionio.api import api_v1 -__version__ = '0.39.20.3' +__version__ = '0.39.20.4' datastore = None From 3c31f023ce1e7698955c0a4f8880143bc78202a8 Mon Sep 17 00:00:00 2001 From: Michael McMillan Date: Tue, 18 Oct 2022 09:16:22 +0200 Subject: [PATCH 10/25] Option to Hide the Referer header from monitored websites. (#996) --- changedetectionio/changedetection.py | 8 ++++++++ docker-compose.yml | 3 +++ 2 files changed, 11 insertions(+) diff --git a/changedetectionio/changedetection.py b/changedetectionio/changedetection.py index 32c21ac46..461476e1b 100755 --- a/changedetectionio/changedetection.py +++ b/changedetectionio/changedetection.py @@ -102,6 +102,14 @@ def main(): has_password=datastore.data['settings']['application']['password'] != False ) + # Monitored websites will not receive a Referer header + # when a user clicks on an outgoing link. + @app.after_request + def hide_referrer(response): + if os.getenv("HIDE_REFERER", False): + response.headers["Referrer-Policy"] = "no-referrer" + return response + # Proxy sub-directory support # Set environment var USE_X_SETTINGS=1 on this script # And then in your proxy_pass settings diff --git a/docker-compose.yml b/docker-compose.yml index 65417ee7f..c04fcf0c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,6 +45,9 @@ services: # Respect proxy_pass type settings, `proxy_set_header Host "localhost";` and `proxy_set_header X-Forwarded-Prefix /app;` # More here https://github.com/dgtlmoon/changedetection.io/wiki/Running-changedetection.io-behind-a-reverse-proxy-sub-directory # - USE_X_SETTINGS=1 + # + # Hides the `Referer` header so that monitored websites can't see the changedetection.io hostname. + # - HIDE_REFERER=true # Comment out ports: when using behind a reverse proxy , enable networks: etc. ports: From 5a43a350dea113ddb0903010e5b7a4546a67e341 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 19 Oct 2022 22:41:13 +0200 Subject: [PATCH 11/25] History index safety check - Be sure that only valid history index lines are read (#1042) --- changedetectionio/model/Watch.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 566eb88e8..5834b532a 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -118,7 +118,10 @@ class model(dict): if os.path.isfile(fname): logging.debug("Reading history index " + str(time.time())) with open(fname, "r") as f: - tmp_history = dict(i.strip().split(',', 2) for i in f.readlines()) + for i in f.readlines(): + if ',' in i: + k, v = i.strip().split(',', 2) + tmp_history[k] = v if len(tmp_history): self.__newest_history_key = list(tmp_history.keys())[-1] From bad0909cc2e40e0b4c280a05b76b4c94a693de4f Mon Sep 17 00:00:00 2001 From: bwees Date: Wed, 19 Oct 2022 18:42:04 -0400 Subject: [PATCH 12/25] added external header server --- changedetectionio/fetch_site_status.py | 15 ++++++++++++++- changedetectionio/forms.py | 1 + changedetectionio/model/Watch.py | 1 + changedetectionio/templates/edit.html | 6 ++++++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index 6c3dbec89..1b1a48275 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -4,7 +4,8 @@ import os import re import time import urllib3 - +import requests +import simplejson from changedetectionio import content_fetcher, html_tools urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -67,6 +68,18 @@ class perform_site_check(): # Tweak the base config with the per-watch ones request_headers = self.datastore.data['settings']['headers'].copy() + + if self.datastore.get_val(uuid, 'external_header_server') is not None: + try: + resp = requests.get(self.datastore.get_val(uuid, 'external_header_server')) + if resp.status_code != 200: + raise Exception("External header server returned non-200 response. Please check the URL for the server") + + request_headers.update(resp.json()) + + except simplejson.errors.JSONDecodeError: + raise Exception("Failed to decode JSON response from external header server") + request_headers.update(extra_headers) # https://github.com/psf/requests/issues/4525 diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 4ad1b1a7e..96ac0e684 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -336,6 +336,7 @@ class watchForm(commonSettingsForm): title = StringField('Title', default='') ignore_text = StringListField('Ignore text', [ValidateListRegex()]) + external_header_server = fields.URLField('External Header Server', validators=[validateURL()]) headers = StringDictKeyValue('Request headers') body = TextAreaField('Request body', [validators.Optional()]) method = SelectField('Request method', choices=valid_method, default=default_method) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index acfd91173..9e436b799 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -25,6 +25,7 @@ class model(dict): 'previous_md5': False, 'uuid': str(uuid_builder.uuid4()), 'headers': {}, # Extra headers to send + 'external_header_server': None, # URL to a server that will return headers 'body': None, 'method': 'GET', #'history': {}, # Dict of timestamp and output stripped filename diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index b13afe46a..0012adbbc 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -109,6 +109,12 @@
{{ render_field(form.method) }}
+
+ {{ render_field(form.external_header_server, placeholder="http://example.com/watch1") }} +
+ The watch will perform a GET request before each check to this URL and will use the headers in addition to the ones listed below and in global settings. More help and examples here +
+
{{ render_field(form.headers, rows=5, placeholder="Example Cookie: foobar From 0d5820932fd12b3e533fd2d18a420225e48dbf78 Mon Sep 17 00:00:00 2001 From: bwees Date: Wed, 19 Oct 2022 18:45:43 -0400 Subject: [PATCH 13/25] rename branch --- changedetectionio/fetch_site_status.py | 1 - 1 file changed, 1 deletion(-) diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index c577c563c..2fb16ed48 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -10,7 +10,6 @@ from changedetectionio import content_fetcher, html_tools urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - # Some common stuff here that can be moved to a base class # (set_proxy_from_list) class perform_site_check(): From 495e322c9ec41b619e30fa9699615ef25f2b9e17 Mon Sep 17 00:00:00 2001 From: bwees Date: Wed, 19 Oct 2022 18:55:05 -0400 Subject: [PATCH 14/25] fixed import errors --- changedetectionio/fetch_site_status.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index 2fb16ed48..61b9a40a3 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -2,10 +2,9 @@ import hashlib import logging import os import re -import time import urllib3 import requests -import simplejson +import json from changedetectionio import content_fetcher, html_tools urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -57,17 +56,18 @@ class perform_site_check(): # Tweak the base config with the per-watch ones request_headers = self.datastore.data['settings']['headers'].copy() - if self.datastore.get_val(uuid, 'external_header_server') is not None: + if self.datastore.data['watching'][uuid].get('external_header_server') is not None: try: - resp = requests.get(self.datastore.get_val(uuid, 'external_header_server')) + resp = requests.get(self.datastore.data['watching'][uuid].get('external_header_server')) if resp.status_code != 200: raise Exception("External header server returned non-200 response. Please check the URL for the server") + data = json.loads(resp.text.strip()) request_headers.update(resp.json()) - except simplejson.errors.JSONDecodeError: + except json.decoder.JSONDecodeError: raise Exception("Failed to decode JSON response from external header server") - + request_headers.update(extra_headers) # https://github.com/psf/requests/issues/4525 From 0a2644d0c37faeb189af9c949a5f8c7e0635a0d4 Mon Sep 17 00:00:00 2001 From: bwees Date: Wed, 19 Oct 2022 18:58:54 -0400 Subject: [PATCH 15/25] fix tests --- changedetectionio/tests/test_auth.py | 2 +- changedetectionio/tests/test_css_selector.py | 2 +- changedetectionio/tests/test_ignorestatuscode.py | 2 +- changedetectionio/tests/test_share_watch.py | 2 +- changedetectionio/tests/test_xpath_selector.py | 10 +++++----- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/changedetectionio/tests/test_auth.py b/changedetectionio/tests/test_auth.py index f8d1437e7..38f897218 100644 --- a/changedetectionio/tests/test_auth.py +++ b/changedetectionio/tests/test_auth.py @@ -23,7 +23,7 @@ def test_basic_auth(client, live_server): # Check form validation res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"css_filter": "", "url": test_url, "tag": "", "headers": "", "external_header_server": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data diff --git a/changedetectionio/tests/test_css_selector.py b/changedetectionio/tests/test_css_selector.py index ab234ddb6..d7d0a1af5 100644 --- a/changedetectionio/tests/test_css_selector.py +++ b/changedetectionio/tests/test_css_selector.py @@ -98,7 +98,7 @@ def test_check_markup_css_filter_restriction(client, live_server): # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data diff --git a/changedetectionio/tests/test_ignorestatuscode.py b/changedetectionio/tests/test_ignorestatuscode.py index aeafcdaaa..15aa1d78b 100644 --- a/changedetectionio/tests/test_ignorestatuscode.py +++ b/changedetectionio/tests/test_ignorestatuscode.py @@ -114,7 +114,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server): # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), - data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data diff --git a/changedetectionio/tests/test_share_watch.py b/changedetectionio/tests/test_share_watch.py index 620bda03a..5f7eff360 100644 --- a/changedetectionio/tests/test_share_watch.py +++ b/changedetectionio/tests/test_share_watch.py @@ -29,7 +29,7 @@ def test_share_watch(client, live_server): # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data diff --git a/changedetectionio/tests/test_xpath_selector.py b/changedetectionio/tests/test_xpath_selector.py index 4e417a745..9be957d6d 100644 --- a/changedetectionio/tests/test_xpath_selector.py +++ b/changedetectionio/tests/test_xpath_selector.py @@ -89,7 +89,7 @@ def test_check_xpath_filter_utf8(client, live_server): time.sleep(1) res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -143,7 +143,7 @@ def test_check_xpath_text_function_utf8(client, live_server): time.sleep(1) res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -192,7 +192,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server): # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"css_filter": xpath_filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -233,7 +233,7 @@ def test_xpath_validation(client, live_server): res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"css_filter": "/something horrible", "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"is not a valid XPath expression" in res.data @@ -263,7 +263,7 @@ def test_check_with_prefix_css_filter(client, live_server): res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"css_filter": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, follow_redirects=True ) From 296c7c46cbc14d242326237ee77612d2cff48277 Mon Sep 17 00:00:00 2001 From: bwees Date: Wed, 19 Oct 2022 19:00:38 -0400 Subject: [PATCH 16/25] fixed empty field errors --- changedetectionio/fetch_site_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index 61b9a40a3..75118ef5b 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -56,7 +56,7 @@ class perform_site_check(): # Tweak the base config with the per-watch ones request_headers = self.datastore.data['settings']['headers'].copy() - if self.datastore.data['watching'][uuid].get('external_header_server') is not None: + if self.datastore.data['watching'][uuid].get('external_header_server') is not None or self.datastore.data['watching'][uuid].get('external_header_server') != "": try: resp = requests.get(self.datastore.data['watching'][uuid].get('external_header_server')) if resp.status_code != 200: From 83161e4fa36467ada9b75eb4c06ac48172706862 Mon Sep 17 00:00:00 2001 From: bwees Date: Wed, 19 Oct 2022 19:03:01 -0400 Subject: [PATCH 17/25] fixed string None case --- changedetectionio/fetch_site_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index 75118ef5b..743c2bc9c 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -56,7 +56,7 @@ class perform_site_check(): # Tweak the base config with the per-watch ones request_headers = self.datastore.data['settings']['headers'].copy() - if self.datastore.data['watching'][uuid].get('external_header_server') is not None or self.datastore.data['watching'][uuid].get('external_header_server') != "": + if self.datastore.data['watching'][uuid].get('external_header_server') is not None or self.datastore.data['watching'][uuid].get('external_header_server') != "" and self.datastore.data['watching'][uuid].get('external_header_server') != "None": try: resp = requests.get(self.datastore.data['watching'][uuid].get('external_header_server')) if resp.status_code != 200: From 76fd27dfabb3d2575e022844ef932db104943a5d Mon Sep 17 00:00:00 2001 From: bwees Date: Wed, 19 Oct 2022 19:10:01 -0400 Subject: [PATCH 18/25] fix logic error --- changedetectionio/fetch_site_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index 743c2bc9c..32d8c8a85 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -56,7 +56,7 @@ class perform_site_check(): # Tweak the base config with the per-watch ones request_headers = self.datastore.data['settings']['headers'].copy() - if self.datastore.data['watching'][uuid].get('external_header_server') is not None or self.datastore.data['watching'][uuid].get('external_header_server') != "" and self.datastore.data['watching'][uuid].get('external_header_server') != "None": + if self.datastore.data['watching'][uuid].get('external_header_server') is not None and self.datastore.data['watching'][uuid].get('external_header_server') != "" and self.datastore.data['watching'][uuid].get('external_header_server') != "None": try: resp = requests.get(self.datastore.data['watching'][uuid].get('external_header_server')) if resp.status_code != 200: From 852a69862959a20c04ae1045ca3cf22e09492a97 Mon Sep 17 00:00:00 2001 From: bwees Date: Wed, 19 Oct 2022 19:14:01 -0400 Subject: [PATCH 19/25] add optional for field --- changedetectionio/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 627c85619..34c54bda2 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -358,7 +358,7 @@ class watchForm(commonSettingsForm): title = StringField('Title', default='') ignore_text = StringListField('Ignore text', [ValidateListRegex()]) - external_header_server = fields.URLField('External Header Server', validators=[validateURL()]) + external_header_server = fields.URLField('External Header Server', validators=[validators.Optional(), validateURL()]) headers = StringDictKeyValue('Request headers') body = TextAreaField('Request body', [validators.Optional()]) method = SelectField('Request method', choices=valid_method, default=default_method) From 9c5588c79100f0fc4dfaa1518d86d197681b5c22 Mon Sep 17 00:00:00 2001 From: Entepotenz <19738301+Entepotenz@users.noreply.github.com> Date: Sun, 23 Oct 2022 11:25:29 +0200 Subject: [PATCH 20/25] update path for validation in the CONTRIBUTING.md (#1046) --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9641dd16d..8478a7aba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ Otherwise, it's always best to PR into the `dev` branch. Please be sure that all new functionality has a matching test! -Use `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notifications.py` for example +Use `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notification.py` for example ``` pip3 install -r requirements-dev From 7839551d6b69af6d16cfff4a18a745de70de96e1 Mon Sep 17 00:00:00 2001 From: Entepotenz <19738301+Entepotenz@users.noreply.github.com> Date: Sun, 23 Oct 2022 11:26:32 +0200 Subject: [PATCH 21/25] Testing - Use same version of playwright while running tests as in production builds (#1047) --- changedetectionio/run_all_tests.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/changedetectionio/run_all_tests.sh b/changedetectionio/run_all_tests.sh index 28dd85c67..4eff9e930 100755 --- a/changedetectionio/run_all_tests.sh +++ b/changedetectionio/run_all_tests.sh @@ -9,6 +9,8 @@ # exit when any command fails set -e +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + find tests/test_*py -type f|while read test_name do echo "TEST RUNNING $test_name" @@ -45,7 +47,9 @@ docker kill $$-test_selenium echo "TESTING WEBDRIVER FETCH > PLAYWRIGHT/BROWSERLESS..." # Not all platforms support playwright (not ARM/rPI), so it's not packaged in requirements.txt -pip3 install playwright~=1.24 +PLAYWRIGHT_VERSION=$(grep -i -E "RUN pip install.+" "$SCRIPT_DIR/../Dockerfile" | grep --only-matching -i -E "playwright[=><~+]+[0-9\.]+") +echo "using $PLAYWRIGHT_VERSION" +pip3 install "$PLAYWRIGHT_VERSION" docker run -d --name $$-test_browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm -p 3000:3000 --shm-size="2g" browserless/chrome:1.53-chrome-stable # takes a while to spin up sleep 5 From 0394a56be57f6278f884908fc38fffcb6ed4f685 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sun, 23 Oct 2022 15:54:19 +0200 Subject: [PATCH 22/25] Building - Test container build on PR --- .github/workflows/test-container-build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-container-build.yml b/.github/workflows/test-container-build.yml index dc6ab712b..cc4572863 100644 --- a/.github/workflows/test-container-build.yml +++ b/.github/workflows/test-container-build.yml @@ -1,8 +1,7 @@ name: ChangeDetection.io Container Build Test # Triggers the workflow on push or pull request events -on: - push: +on: [push, pull_request] paths: - requirements.txt - Dockerfile From 492bbce6b67908d1b7557f58642746d1cb3519bc Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sun, 23 Oct 2022 16:02:13 +0200 Subject: [PATCH 23/25] Build - Fix syntax in container build test (#1050) --- .github/workflows/test-container-build.yml | 12 +++++++++++- Dockerfile | 1 + requirements.txt | 7 ++++--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-container-build.yml b/.github/workflows/test-container-build.yml index cc4572863..7d59ad0b0 100644 --- a/.github/workflows/test-container-build.yml +++ b/.github/workflows/test-container-build.yml @@ -1,7 +1,17 @@ name: ChangeDetection.io Container Build Test # Triggers the workflow on push or pull request events -on: [push, pull_request] + +# This line doesnt work, even tho it is the documented one +#on: [push, pull_request] + +on: + push: + paths: + - requirements.txt + - Dockerfile + + pull_request: paths: - requirements.txt - Dockerfile diff --git a/Dockerfile b/Dockerfile index d422918e6..978a912cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,6 +64,7 @@ EXPOSE 5000 # The actual flask app COPY changedetectionio /app/changedetectionio + # The eventlet server wrapper COPY changedetection.py /app/changedetection.py diff --git a/requirements.txt b/requirements.txt index bffc2a7f3..500f45f9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -flask~= 2.0 +flask ~= 2.0 flask_wtf -eventlet>=0.31.0 +eventlet >= 0.31.0 validators -timeago ~=1.0 +timeago ~= 1.0 inscriptis ~= 2.2 feedgen ~= 0.9 flask-login ~= 0.5 @@ -47,3 +47,4 @@ selenium ~= 4.1.0 werkzeug ~= 2.0.0 # playwright is installed at Dockerfile build time because it's not available on all platforms + From 5d40e16c73b74888a19abecb911e01156d0172de Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sun, 23 Oct 2022 19:15:11 +0200 Subject: [PATCH 24/25] API - Adding basic system info/system state API (#1051) --- changedetectionio/__init__.py | 3 +++ changedetectionio/api/api_v1.py | 30 +++++++++++++++++++++++++++++ changedetectionio/store.py | 6 +++--- changedetectionio/tests/test_api.py | 10 ++++++++++ 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index c6f95f1ea..8bbb747d0 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -194,6 +194,9 @@ def changedetection_app(config=None, datastore_o=None): watch_api.add_resource(api_v1.Watch, '/api/v1/watch/', resource_class_kwargs={'datastore': datastore, 'update_q': update_q}) + watch_api.add_resource(api_v1.SystemInfo, '/api/v1/systeminfo', + resource_class_kwargs={'datastore': datastore, 'update_q': update_q}) + diff --git a/changedetectionio/api/api_v1.py b/changedetectionio/api/api_v1.py index a432bc670..d44c990ed 100644 --- a/changedetectionio/api/api_v1.py +++ b/changedetectionio/api/api_v1.py @@ -122,3 +122,33 @@ class CreateWatch(Resource): return {'status': "OK"}, 200 return list, 200 + +class SystemInfo(Resource): + def __init__(self, **kwargs): + # datastore is a black box dependency + self.datastore = kwargs['datastore'] + self.update_q = kwargs['update_q'] + + @auth.check_token + def get(self): + import time + overdue_watches = [] + + # Check all watches and report which have not been checked but should have been + + for uuid, watch in self.datastore.data.get('watching', {}).items(): + # see if now - last_checked is greater than the time that should have been + # this is not super accurate (maybe they just edited it) but better than nothing + t = watch.threshold_seconds() + if not t: + t = self.datastore.threshold_seconds + time_since_check = time.time() - watch.get('last_checked') + if time_since_check > t: + overdue_watches.append(uuid) + + return { + 'queue_size': self.update_q.qsize(), + 'overdue_watches': overdue_watches, + 'uptime': round(time.time() - self.datastore.start_time, 2), + 'watch_count': len(self.datastore.data.get('watching', {})) + }, 200 diff --git a/changedetectionio/store.py b/changedetectionio/store.py index bd86039ad..6182aef86 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -30,14 +30,14 @@ class ChangeDetectionStore: def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"): # Should only be active for docker # logging.basicConfig(filename='/dev/stdout', level=logging.INFO) - self.needs_write = False + self.__data = App.model() self.datastore_path = datastore_path self.json_store_path = "{}/url-watches.json".format(self.datastore_path) + self.needs_write = False self.proxy_list = None + self.start_time = time.time() self.stop_thread = False - self.__data = App.model() - # Base definition for all watchers # deepcopy part of #569 - not sure why its needed exactly self.generic_definition = deepcopy(Watch.model(datastore_path = datastore_path, default={})) diff --git a/changedetectionio/tests/test_api.py b/changedetectionio/tests/test_api.py index dd66012e3..504a9554c 100644 --- a/changedetectionio/tests/test_api.py +++ b/changedetectionio/tests/test_api.py @@ -147,6 +147,16 @@ def test_api_simple(client, live_server): # @todo how to handle None/default global values? assert watch['history_n'] == 2, "Found replacement history section, which is in its own API" + # basic systeminfo check + res = client.get( + url_for("systeminfo"), + headers={'x-api-key': api_key}, + ) + info = json.loads(res.data) + assert info.get('watch_count') == 1 + assert info.get('uptime') > 0.5 + + # Finally delete the watch res = client.delete( url_for("watch", uuid=watch_uuid), From 4eb4b401a1891c6af359abc0a58b243cf58acc19 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sun, 23 Oct 2022 23:12:28 +0200 Subject: [PATCH 25/25] API - system info - allow 5 minutes grace before watch is considered 'overdue' --- changedetectionio/api/api_v1.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/changedetectionio/api/api_v1.py b/changedetectionio/api/api_v1.py index d44c990ed..40131ca52 100644 --- a/changedetectionio/api/api_v1.py +++ b/changedetectionio/api/api_v1.py @@ -141,9 +141,13 @@ class SystemInfo(Resource): # this is not super accurate (maybe they just edited it) but better than nothing t = watch.threshold_seconds() if not t: + # Use the system wide default t = self.datastore.threshold_seconds + time_since_check = time.time() - watch.get('last_checked') - if time_since_check > t: + + # Allow 5 minutes of grace time before we decide it's overdue + if time_since_check - (5 * 60) > t: overdue_watches.append(uuid) return {