Merge branch 'master' into fix-mixed-html-alerts

.github/workflows/pypi-release.yml

@@ -45,7 +45,6 @@ jobs:
     - name: Test that the basic pip built package runs without error
       run: |
         set -ex
-        sudo pip3 install --upgrade pip 
         pip3 install dist/changedetection.io*.whl
         changedetection.io -d /tmp -p 10000 &
         sleep 3

.github/workflows/test-stack-reusable-workflow.yml

@@ -64,14 +64,16 @@ jobs:
           echo "Running processes in docker..."
           docker ps
 
-      - name: Test built container with Pytest (generally as requests/plaintext fetching)
+      - name: Run Unit Tests
         run: |
           # Unit tests
-          echo "run test with unittest"
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
-          
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
+
+      - name: Test built container with Pytest (generally as requests/plaintext fetching)
+        run: |
           # All tests
           echo "run test with pytest"
           # The default pytest logger_level is TRACE

README.md

@@ -120,7 +120,7 @@ Easily add the current web page to your changedetection.io tool, simply install
 
 [<img src="./docs/chrome-extension-screenshot.png" style="max-width:80%;" alt="Chrome Extension to easily add the current web-page to detect a change."  title="Chrome Extension to easily add the current web-page to detect a change."  />](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
 
-[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
+[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop) ( Or check out the [GitHub repo](https://github.com/dgtlmoon/changedetection.io-browser-extension) ) 
 
 ## Installation
 

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.48.05'
+__version__ = '0.49.0'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
@@ -24,6 +24,9 @@ from loguru import logger
 app = None
 datastore = None
 
+def get_version():
+    return __version__
+
 # Parent wrapper or OS sends us a SIGTERM/SIGINT, do everything required for a clean shutdown
 def sigshutdown_handler(_signo, _stack_frame):
     global app

changedetectionio/apprise_plugin/__init__.py

@@ -27,19 +27,17 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     method =  re.sub(rf's$', '', schema)
     requests_method = getattr(requests, method)
 
-    headers = CaseInsensitiveDict({})
     params = CaseInsensitiveDict({}) # Added to requests
     auth = None
     has_error = False
 
-
     # Convert /foobar?+some-header=hello to proper header dictionary
     results = apprise_parse_url(url)
 
     # Add our headers that the user can potentially over-ride if they wish
     # to to our returned result set and tidy entries by unquoting them
-    headers = {unquote_plus(x): unquote_plus(y)
-               for x, y in results['qsd+'].items()}
+    headers = CaseInsensitiveDict({unquote_plus(x): unquote_plus(y)
+               for x, y in results['qsd+'].items()})
 
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
     # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
@@ -81,7 +79,7 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
           params=params
         )
 
-        if r.status_code not in (requests.codes.created, requests.codes.ok):
+        if not (200 <= r.status_code < 300):
             status_str = f"Error sending '{method.upper()}' request to {url} - Status: {r.status_code}: '{r.reason}'"
             logger.error(status_str)
             has_error = True

changedetectionio/model/App.py

@@ -69,7 +69,7 @@ def parse_headers_from_text_file(filepath):
         for l in f.readlines():
             l = l.strip()
             if not l.startswith('#') and ':' in l:
-                (k, v) = l.split(':')
+                (k, v) = l.split(':', 1)  # Split only on the first colon
                 headers[k.strip()] = v.strip()
 
     return headers

changedetectionio/tests/test_notification.py

@@ -29,7 +29,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
     # Re 360 - new install should have defaults set
     res = client.get(url_for("settings_page"))
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')+"?status_code=204"
 
     assert default_notification_body.encode() in res.data
     assert default_notification_title.encode() in res.data
@@ -135,7 +135,14 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
     time.sleep(3)
+
+    # Check no errors were recorded
+    res = client.get(url_for("index"))
+    assert b'notification-error' not in res.data
+
+
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
         notification_submission = f.read()
@@ -284,7 +291,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
 
     res = client.post(
         url_for("settings_page"),
@@ -319,6 +326,11 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
 
     time.sleep(2) # plus extra delay for notifications to fire
 
+
+    # Check no errors were recorded, because we asked for 204 which is slightly uncommon but is still OK
+    res = client.get(url_for("index"))
+    assert b'notification-error' not in res.data
+
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
         j = json.loads(x)

changedetectionio/tests/test_request.py

@@ -373,13 +373,13 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     with open('test-datastore/headers-testtag.txt', 'w') as f:
-        f.write("tag-header: test")
+        f.write("tag-header: test\r\nurl-header: http://example.com")
 
     with open('test-datastore/headers.txt', 'w') as f:
-        f.write("global-header: nice\r\nnext-global-header: nice")
+        f.write("global-header: nice\r\nnext-global-header: nice\r\nurl-header-global: http://example.com/global")
 
     with open('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt', 'w') as f:
-        f.write("watch-header: nice")
+        f.write("watch-header: nice\r\nurl-header-watch: http://example.com/watch")
 
     wait_for_all_checks(client)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -410,6 +410,9 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     assert b"Xxx:ooo" in res.data
     assert b"Watch-Header:nice" in res.data
     assert b"Tag-Header:test" in res.data
+    assert b"Url-Header:http://example.com" in res.data
+    assert b"Url-Header-Global:http://example.com/global" in res.data
+    assert b"Url-Header-Watch:http://example.com/watch" in res.data
 
     # Check the custom UA from system settings page made it through
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):

changedetectionio/tests/test_restock_itemprop.py

@@ -189,6 +189,17 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
 
     client.get(url_for("mark_all_viewed"))
 
+
+    # 2715 - Price detection (once it crosses the "lower" threshold) again with a lower price - should trigger again!
+    set_original_response(props_markup=instock_props[0], price='820.45')
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    assert b'1 watches queued for rechecking.' in res.data
+    wait_for_all_checks(client)
+    res = client.get(url_for("index"))
+    assert b'820.45' in res.data
+    assert b'unviewed' in res.data
+    client.get(url_for("mark_all_viewed"))
+
     # price changed to something MORE than max (1100.10), SHOULD be a change
     set_original_response(props_markup=instock_props[0], price='1890.45')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -203,7 +214,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
 
 
 def test_restock_itemprop_minmax(client, live_server):
-#    live_server_setup(live_server)
+    #live_server_setup(live_server)
     extras = {
         "restock_settings-follow_price_changes": "y",
         "restock_settings-price_change_min": 900.0,

changedetectionio/tests/unit/test_semver.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+# run from dir above changedetectionio/ dir
+# python3 -m unittest changedetectionio.tests.unit.test_semver
+
+import re
+import unittest
+
+
+# The SEMVER regex
+SEMVER_REGEX = r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
+
+# Compile the regex
+semver_pattern = re.compile(SEMVER_REGEX)
+
+class TestSemver(unittest.TestCase):
+    def test_valid_versions(self):
+        """Test valid semantic version strings"""
+        valid_versions = [
+            "1.0.0",
+            "0.1.0",
+            "0.0.1",
+            "1.0.0-alpha",
+            "1.0.0-alpha.1",
+            "1.0.0-0.3.7",
+            "1.0.0-x.7.z.92",
+            "1.0.0-alpha+001",
+            "1.0.0+20130313144700",
+            "1.0.0-beta+exp.sha.5114f85"
+        ]
+        for version in valid_versions:
+            with self.subTest(version=version):
+                self.assertIsNotNone(semver_pattern.match(version), f"Version {version} should be valid")
+
+    def test_invalid_versions(self):
+        """Test invalid semantic version strings"""
+        invalid_versions = [
+            "0.48.06",
+            "1.0",
+            "1.0.0-",
+# Seems to pass the semver.org regex?
+#            "1.0.0-alpha-",
+            "1.0.0+",
+            "1.0.0-alpha+",
+            "1.0.0-",
+            "01.0.0",
+            "1.01.0",
+            "1.0.01",
+            ".1.0.0",
+            "1..0.0"
+        ]
+        for version in invalid_versions:
+            with self.subTest(version=version):
+                res = semver_pattern.match(version)
+                self.assertIsNone(res, f"Version '{version}' should be invalid")
+
+    def test_our_version(self):
+        from changedetectionio import get_version
+        our_version = get_version()
+        self.assertIsNotNone(semver_pattern.match(our_version), f"Our version '{our_version}' should be a valid SEMVER string")
+
+
+if __name__ == '__main__':
+    unittest.main()

changedetectionio/tests/util.py

@@ -244,8 +244,11 @@ def live_server_setup(live_server):
                 f.write(request.content_type)
 
         print("\n>> Test notification endpoint was hit.\n", data)
-        return "Text was set"
 
+        content = "Text was set"
+        status_code = request.args.get('status_code',200)
+        resp = make_response(content, status_code)
+        return resp
 
     # Just return the verb in the request
     @live_server.app.route('/test-basicauth', methods=['GET'])

requirements.txt

@@ -95,5 +95,8 @@ babel
 # Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096
 greenlet >= 3.0.3
 
+# Pinned or it causes problems with flask_expects_json which seems unmaintained
+referencing==0.35.1
+
 # Scheduler - Windows seemed to miss a lot of default timezone info (even "UTC" !)
 tzdata