From a2a98811a5645495c967a903abf902ac8f1ecf6c Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 9 Jan 2025 22:59:55 +0100 Subject: [PATCH 1/7] Restock - Add test for new lower/higher price notification Re #2715 (#2892) --- changedetectionio/tests/test_restock_itemprop.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/changedetectionio/tests/test_restock_itemprop.py b/changedetectionio/tests/test_restock_itemprop.py index 057099ad5..4a9087121 100644 --- a/changedetectionio/tests/test_restock_itemprop.py +++ b/changedetectionio/tests/test_restock_itemprop.py @@ -189,6 +189,17 @@ def _run_test_minmax_limit(client, extra_watch_edit_form): client.get(url_for("mark_all_viewed")) + + # 2715 - Price detection (once it crosses the "lower" threshold) again with a lower price - should trigger again! + set_original_response(props_markup=instock_props[0], price='820.45') + res = client.get(url_for("form_watch_checknow"), follow_redirects=True) + assert b'1 watches queued for rechecking.' in res.data + wait_for_all_checks(client) + res = client.get(url_for("index")) + assert b'820.45' in res.data + assert b'unviewed' in res.data + client.get(url_for("mark_all_viewed")) + # price changed to something MORE than max (1100.10), SHOULD be a change set_original_response(props_markup=instock_props[0], price='1890.45') client.get(url_for("form_watch_checknow"), follow_redirects=True) @@ -203,7 +214,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form): def test_restock_itemprop_minmax(client, live_server): -# live_server_setup(live_server) + #live_server_setup(live_server) extras = { "restock_settings-follow_price_changes": "y", "restock_settings-price_change_min": 900.0, From 1c1f1c6f6bd5af7e6ca4e72848edda7ca51d0ec6 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 9 Jan 2025 23:02:29 +0100 Subject: [PATCH 2/7] 0.48.06 --- changedetectionio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index f8c2c1614..f30d95e4a 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -2,7 +2,7 @@ # Read more https://github.com/dgtlmoon/changedetection.io/wiki -__version__ = '0.48.05' +__version__ = '0.48.06' from changedetectionio.strtobool import strtobool from json.decoder import JSONDecodeError From 8960f401b71d02519267819cfc617b8cd7305d4c Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 13 Jan 2025 13:13:18 +0100 Subject: [PATCH 3/7] Notifications - Custom POST:// GET:// etc endpoints - returning 204 and other 20x responses are OK (don't show an error was detected)(#2897) --- changedetectionio/apprise_plugin/__init__.py | 8 +++----- changedetectionio/tests/test_notification.py | 16 ++++++++++++++-- changedetectionio/tests/util.py | 5 ++++- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/changedetectionio/apprise_plugin/__init__.py b/changedetectionio/apprise_plugin/__init__.py index ad8392cab..dc4346733 100644 --- a/changedetectionio/apprise_plugin/__init__.py +++ b/changedetectionio/apprise_plugin/__init__.py @@ -27,19 +27,17 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs): method = re.sub(rf's$', '', schema) requests_method = getattr(requests, method) - headers = CaseInsensitiveDict({}) params = CaseInsensitiveDict({}) # Added to requests auth = None has_error = False - # Convert /foobar?+some-header=hello to proper header dictionary results = apprise_parse_url(url) # Add our headers that the user can potentially over-ride if they wish # to to our returned result set and tidy entries by unquoting them - headers = {unquote_plus(x): unquote_plus(y) - for x, y in results['qsd+'].items()} + headers = CaseInsensitiveDict({unquote_plus(x): unquote_plus(y) + for x, y in results['qsd+'].items()}) # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise @@ -81,7 +79,7 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs): params=params ) - if r.status_code not in (requests.codes.created, requests.codes.ok): + if not (200 <= r.status_code < 300): status_str = f"Error sending '{method.upper()}' request to {url} - Status: {r.status_code}: '{r.reason}'" logger.error(status_str) has_error = True diff --git a/changedetectionio/tests/test_notification.py b/changedetectionio/tests/test_notification.py index 0c20a9f10..ceae292ed 100644 --- a/changedetectionio/tests/test_notification.py +++ b/changedetectionio/tests/test_notification.py @@ -29,7 +29,7 @@ def test_check_notification(client, live_server, measure_memory_usage): # Re 360 - new install should have defaults set res = client.get(url_for("settings_page")) - notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json') + notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')+"?status_code=204" assert default_notification_body.encode() in res.data assert default_notification_title.encode() in res.data @@ -135,7 +135,14 @@ def test_check_notification(client, live_server, measure_memory_usage): # Trigger a check client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) time.sleep(3) + + # Check no errors were recorded + res = client.get(url_for("index")) + assert b'notification-error' not in res.data + + # Verify what was sent as a notification, this file should exist with open("test-datastore/notification.txt", "r") as f: notification_submission = f.read() @@ -284,7 +291,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me # CUSTOM JSON BODY CHECK for POST:// set_original_response() # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation - test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22" + test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22" res = client.post( url_for("settings_page"), @@ -319,6 +326,11 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me time.sleep(2) # plus extra delay for notifications to fire + + # Check no errors were recorded, because we asked for 204 which is slightly uncommon but is still OK + res = client.get(url_for("index")) + assert b'notification-error' not in res.data + with open("test-datastore/notification.txt", 'r') as f: x = f.read() j = json.loads(x) diff --git a/changedetectionio/tests/util.py b/changedetectionio/tests/util.py index 9353155e7..250f6742e 100644 --- a/changedetectionio/tests/util.py +++ b/changedetectionio/tests/util.py @@ -244,8 +244,11 @@ def live_server_setup(live_server): f.write(request.content_type) print("\n>> Test notification endpoint was hit.\n", data) - return "Text was set" + content = "Text was set" + status_code = request.args.get('status_code',200) + resp = make_response(content, status_code) + return resp # Just return the verb in the request @live_server.app.route('/test-basicauth', methods=['GET']) From 4f2d38ff49644d79d05c3661256c68ad3bcc5e63 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sat, 18 Jan 2025 23:20:58 +0100 Subject: [PATCH 4/7] Build/Libraries - Pin `referencing` library which breaks due to out-dated flask_expects_json, remove pip upgrade in test(#2912) --- .github/workflows/pypi-release.yml | 1 - requirements.txt | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index b708c9a8b..3c27c2dab 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -45,7 +45,6 @@ jobs: - name: Test that the basic pip built package runs without error run: | set -ex - sudo pip3 install --upgrade pip pip3 install dist/changedetection.io*.whl changedetection.io -d /tmp -p 10000 & sleep 3 diff --git a/requirements.txt b/requirements.txt index 0bcb6552b..3355e770b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -95,5 +95,8 @@ babel # Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096 greenlet >= 3.0.3 +# Pinned or it causes problems with flask_expects_json which seems unmaintained +referencing==0.35.1 + # Scheduler - Windows seemed to miss a lot of default timezone info (even "UTC" !) tzdata From 938065db6f8a735e562e2f62a7fb5300b734efd8 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 20 Jan 2025 16:10:54 +0100 Subject: [PATCH 5/7] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 12bcb507b..4492fcfd7 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ Easily add the current web page to your changedetection.io tool, simply install [Chrome Extension to easily add the current web-page to detect a change.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop) -[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop) +[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop) ( Or check out the [GitHub repo](https://github.com/dgtlmoon/changedetection.io-browser-extension) ) ## Installation From e799a1cdcbf1cb1feaa12838b7a8a3a85fee4f62 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 21 Jan 2025 13:40:01 +0100 Subject: [PATCH 6/7] 0.49.00 --- .../test-stack-reusable-workflow.yml | 8 ++- changedetectionio/__init__.py | 5 +- changedetectionio/tests/unit/test_semver.py | 64 +++++++++++++++++++ 3 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 changedetectionio/tests/unit/test_semver.py diff --git a/.github/workflows/test-stack-reusable-workflow.yml b/.github/workflows/test-stack-reusable-workflow.yml index f2864680c..29903c171 100644 --- a/.github/workflows/test-stack-reusable-workflow.yml +++ b/.github/workflows/test-stack-reusable-workflow.yml @@ -64,14 +64,16 @@ jobs: echo "Running processes in docker..." docker ps - - name: Test built container with Pytest (generally as requests/plaintext fetching) + - name: Run Unit Tests run: | # Unit tests - echo "run test with unittest" docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff' docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model' docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security' - + docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver' + + - name: Test built container with Pytest (generally as requests/plaintext fetching) + run: | # All tests echo "run test with pytest" # The default pytest logger_level is TRACE diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index f30d95e4a..6f386b7e2 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -2,7 +2,7 @@ # Read more https://github.com/dgtlmoon/changedetection.io/wiki -__version__ = '0.48.06' +__version__ = '0.49.0' from changedetectionio.strtobool import strtobool from json.decoder import JSONDecodeError @@ -24,6 +24,9 @@ from loguru import logger app = None datastore = None +def get_version(): + return __version__ + # Parent wrapper or OS sends us a SIGTERM/SIGINT, do everything required for a clean shutdown def sigshutdown_handler(_signo, _stack_frame): global app diff --git a/changedetectionio/tests/unit/test_semver.py b/changedetectionio/tests/unit/test_semver.py new file mode 100644 index 000000000..5ebfc95e7 --- /dev/null +++ b/changedetectionio/tests/unit/test_semver.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +# run from dir above changedetectionio/ dir +# python3 -m unittest changedetectionio.tests.unit.test_semver + +import re +import unittest + + +# The SEMVER regex +SEMVER_REGEX = r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$" + +# Compile the regex +semver_pattern = re.compile(SEMVER_REGEX) + +class TestSemver(unittest.TestCase): + def test_valid_versions(self): + """Test valid semantic version strings""" + valid_versions = [ + "1.0.0", + "0.1.0", + "0.0.1", + "1.0.0-alpha", + "1.0.0-alpha.1", + "1.0.0-0.3.7", + "1.0.0-x.7.z.92", + "1.0.0-alpha+001", + "1.0.0+20130313144700", + "1.0.0-beta+exp.sha.5114f85" + ] + for version in valid_versions: + with self.subTest(version=version): + self.assertIsNotNone(semver_pattern.match(version), f"Version {version} should be valid") + + def test_invalid_versions(self): + """Test invalid semantic version strings""" + invalid_versions = [ + "0.48.06", + "1.0", + "1.0.0-", +# Seems to pass the semver.org regex? +# "1.0.0-alpha-", + "1.0.0+", + "1.0.0-alpha+", + "1.0.0-", + "01.0.0", + "1.01.0", + "1.0.01", + ".1.0.0", + "1..0.0" + ] + for version in invalid_versions: + with self.subTest(version=version): + res = semver_pattern.match(version) + self.assertIsNone(res, f"Version '{version}' should be invalid") + + def test_our_version(self): + from changedetectionio import get_version + our_version = get_version() + self.assertIsNotNone(semver_pattern.match(our_version), f"Our version '{our_version}' should be a valid SEMVER string") + + +if __name__ == '__main__': + unittest.main() From 1c61b5a6239c9e4632492b7950db282e1466f933 Mon Sep 17 00:00:00 2001 From: Iftekhar Alam Fuad Date: Sun, 26 Jan 2025 05:08:09 +0600 Subject: [PATCH 7/7] Header handling - Fix header parsing to split on the first colon only (headers where the value contained :// type may have been broken) (#2929) --- changedetectionio/model/App.py | 2 +- changedetectionio/tests/test_request.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/changedetectionio/model/App.py b/changedetectionio/model/App.py index b70a7ff00..4c9c34fec 100644 --- a/changedetectionio/model/App.py +++ b/changedetectionio/model/App.py @@ -69,7 +69,7 @@ def parse_headers_from_text_file(filepath): for l in f.readlines(): l = l.strip() if not l.startswith('#') and ':' in l: - (k, v) = l.split(':') + (k, v) = l.split(':', 1) # Split only on the first colon headers[k.strip()] = v.strip() return headers \ No newline at end of file diff --git a/changedetectionio/tests/test_request.py b/changedetectionio/tests/test_request.py index e3511f81a..88816308f 100644 --- a/changedetectionio/tests/test_request.py +++ b/changedetectionio/tests/test_request.py @@ -373,13 +373,13 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage): wait_for_all_checks(client) with open('test-datastore/headers-testtag.txt', 'w') as f: - f.write("tag-header: test") + f.write("tag-header: test\r\nurl-header: http://example.com") with open('test-datastore/headers.txt', 'w') as f: - f.write("global-header: nice\r\nnext-global-header: nice") + f.write("global-header: nice\r\nnext-global-header: nice\r\nurl-header-global: http://example.com/global") with open('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt', 'w') as f: - f.write("watch-header: nice") + f.write("watch-header: nice\r\nurl-header-watch: http://example.com/watch") wait_for_all_checks(client) client.get(url_for("form_watch_checknow"), follow_redirects=True) @@ -410,6 +410,9 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage): assert b"Xxx:ooo" in res.data assert b"Watch-Header:nice" in res.data assert b"Tag-Header:test" in res.data + assert b"Url-Header:http://example.com" in res.data + assert b"Url-Header-Global:http://example.com/global" in res.data + assert b"Url-Header-Watch:http://example.com/watch" in res.data # Check the custom UA from system settings page made it through if os.getenv('PLAYWRIGHT_DRIVER_URL'):