Re #2742 fixing URL encoding of headers so that '+' in URL is correctly parsed as ' ' (and other url-encodings)

.gitignore

@@ -10,6 +10,5 @@ dist
 venv
 test-datastore/*
 test-datastore
-test-memory.log
 *.egg-info*
 .vscode/settings.json

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.47.04'
+__version__ = '0.47.03'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -30,8 +30,6 @@ function isItemInStock() {
         'dieser artikel ist bald wieder verfügbar',
         'dostępne wkrótce',
         'en rupture de stock',
-        'esgotado',
-        'indisponível',
         'isn\'t in stock right now',
         'isnt in stock right now',
         'isn’t in stock right now',
@@ -59,7 +57,6 @@ function isItemInStock() {
         'notify me when available',
         'notify me',
         'notify when available',
-        'não disponível',
         'não estamos a aceitar encomendas',
         'out of stock',
         'out-of-stock',

changedetectionio/flask_app.py

@@ -537,27 +537,21 @@ def changedetection_app(config=None, datastore_o=None):
         import random
         from .apprise_asset import asset
         apobj = apprise.Apprise(asset=asset)
-
         # so that the custom endpoints are registered
         from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper
         is_global_settings_form = request.args.get('mode', '') == 'global-settings'
         is_group_settings_form = request.args.get('mode', '') == 'group-settings'
-
         # Use an existing random one on the global/main settings form
         if not watch_uuid and (is_global_settings_form or is_group_settings_form) \
                 and datastore.data.get('watching'):
+
             logger.debug(f"Send test notification - Choosing random Watch {watch_uuid}")
             watch_uuid = random.choice(list(datastore.data['watching'].keys()))
+            watch = datastore.data['watching'].get(watch_uuid)
+        else:
+            watch = None
 
-        if not watch_uuid:
-            return make_response("Error: You must have atleast one watch configured for 'test notification' to work", 400)
-
-        watch = datastore.data['watching'].get(watch_uuid)
-
-        notification_urls = None
-
-        if request.form.get('notification_urls'):
-            notification_urls = request.form['notification_urls'].strip().splitlines()
+        notification_urls = request.form['notification_urls'].strip().splitlines()
 
         if not notification_urls:
             logger.debug("Test notification - Trying by group/tag in the edit form if available")
@@ -575,12 +569,12 @@ def changedetection_app(config=None, datastore_o=None):
 
 
         if not notification_urls:
-            return 'Error: No Notification URLs set/found'
+            return 'No Notification URLs set/found'
 
         for n_url in notification_urls:
             if len(n_url.strip()):
                 if not apobj.add(n_url):
-                    return f'Error:  {n_url} is not a valid AppRise URL.'
+                    return f'Error - {n_url} is not a valid AppRise URL.'
 
         try:
             # use the same as when it is triggered, but then override it with the form test values
@@ -599,13 +593,11 @@ def changedetection_app(config=None, datastore_o=None):
             if 'notification_body' in request.form and request.form['notification_body'].strip():
                 n_object['notification_body'] = request.form.get('notification_body', '').strip()
 
-            n_object.update(watch.extra_notification_token_values())
-
             from . import update_worker
             new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
             new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch)
         except Exception as e:
-            return make_response(f"Error: str(e)", 400)
+            return make_response({'error': str(e)}, 400)
 
         return 'OK - Sent test notifications'
 
@@ -803,9 +795,8 @@ def changedetection_app(config=None, datastore_o=None):
             # But in the case something is added we should save straight away
             datastore.needs_write_urgent = True
 
-            if not datastore.data['watching'][uuid].get('paused'):
-                # Queue the watch for immediate recheck, with a higher priority
-                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+            # Queue the watch for immediate recheck, with a higher priority
+            update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
 
             # Diff page [edit] link should go back to diff page
             if request.args.get("next") and request.args.get("next") == 'diff':

changedetectionio/forms.py

@@ -515,7 +515,6 @@ class processor_text_json_diff_form(commonSettingsForm):
         if not super().validate():
             return False
 
-        from changedetectionio.safe_jinja import render as jinja_render
         result = True
 
         # Fail form validation when a body is set for a GET
@@ -525,46 +524,18 @@ class processor_text_json_diff_form(commonSettingsForm):
 
         # Attempt to validate jinja2 templates in the URL
         try:
+            from changedetectionio.safe_jinja import render as jinja_render
             jinja_render(template_str=self.url.data)
         except ModuleNotFoundError as e:
             # incase jinja2_time or others is missing
             logger.error(e)
-            self.url.errors.append(f'Invalid template syntax configuration: {e}')
+            self.url.errors.append(e)
             result = False
         except Exception as e:
             logger.error(e)
-            self.url.errors.append(f'Invalid template syntax: {e}')
+            self.url.errors.append('Invalid template syntax')
             result = False
 
-        # Attempt to validate jinja2 templates in the body
-        if self.body.data and self.body.data.strip():
-            try:
-                jinja_render(template_str=self.body.data)
-            except ModuleNotFoundError as e:
-                # incase jinja2_time or others is missing
-                logger.error(e)
-                self.body.errors.append(f'Invalid template syntax configuration: {e}')
-                result = False
-            except Exception as e:
-                logger.error(e)
-                self.body.errors.append(f'Invalid template syntax: {e}')
-                result = False
-
-        # Attempt to validate jinja2 templates in the headers
-        if len(self.headers.data) > 0:
-            try:
-                for header, value in self.headers.data.items():
-                    jinja_render(template_str=value)
-            except ModuleNotFoundError as e:
-                # incase jinja2_time or others is missing
-                logger.error(e)
-                self.headers.errors.append(f'Invalid template syntax configuration: {e}')
-                result = False
-            except Exception as e:
-                logger.error(e)
-                self.headers.errors.append(f'Invalid template syntax in "{header}" header: {e}')
-                result = False
-
         return result
 
 class SingleExtraProxy(Form):

changedetectionio/processors/__init__.py

@@ -102,7 +102,6 @@ class difference_detection_processor():
             self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid'))
 
         # Tweak the base config with the per-watch ones
-        from changedetectionio.safe_jinja import render as jinja_render
         request_headers = CaseInsensitiveDict()
 
         ua = self.datastore.data['settings']['requests'].get('default_ua')
@@ -119,15 +118,9 @@ class difference_detection_processor():
         if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
             request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
 
-        for header_name in request_headers:
-            request_headers.update({header_name: jinja_render(template_str=request_headers.get(header_name))})
-
         timeout = self.datastore.data['settings']['requests'].get('timeout')
 
         request_body = self.watch.get('body')
-        if request_body:
-            request_body = jinja_render(template_str=self.watch.get('body'))
-        
         request_method = self.watch.get('method')
         ignore_status_codes = self.watch.get('ignore_status_codes', False)
 

changedetectionio/static/js/notifications.js

@@ -28,14 +28,17 @@ $(document).ready(function() {
       url: notification_base_url,
       data : data,
         statusCode: {
-        400: function(data) {
-          // More than likely the CSRF token was lost when the server restarted
-          alert(data.responseText);
+        400: function() {
+            // More than likely the CSRF token was lost when the server restarted
+          alert("There was a problem processing the request, please reload the page.");
         }
       }
     }).done(function(data){
       console.log(data);
       alert(data);
+    }).fail(function(data){
+      console.log(data);
+      alert('There was an error communicating with the server.');
     })
   });
 });

changedetectionio/templates/edit.html

@@ -65,8 +65,8 @@
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
-                        <div class="pure-form-message">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></div>
-                        <div class="pure-form-message">Variables are supported in the URL (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a>).</div>
+                        <span class="pure-form-message-inline">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></span><br>
+                        <span class="pure-form-message-inline">You can use variables in the URL, perfect for inserting the current date and other logic, <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a></span><br>
                     </div>
                     <div class="pure-control-group inline-radio">
                         {{ render_field(form.processor) }}
@@ -149,24 +149,21 @@
                             {{ render_field(form.method) }}
                         </div>
                         <div id="request-body">
-                                            {{ render_field(form.body, rows=7, placeholder="Example
+                                            {{ render_field(form.body, rows=5, placeholder="Example
 {
    \"name\":\"John\",
    \"age\":30,
-   \"car\":null,
-   \"year\":{% now 'Europe/Berlin', '%Y' %}
+   \"car\":null
 }") }}
                         </div>
-                        <div class="pure-form-message">Variables are supported in the request body (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a>).</div>
                     </div>
                 </fieldset>
             <!-- hmm -->
                 <div class="pure-control-group advanced-options"  style="display: none;">
-                    {{ render_field(form.headers, rows=7, placeholder="Example
+                    {{ render_field(form.headers, rows=5, placeholder="Example
 Cookie: foobar
-User-Agent: wonderbra 1.0
-Math: {{ 1 + 1 }}") }}
-                        <div class="pure-form-message">Variables are supported in the request header values (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a>).</div>
+User-Agent: wonderbra 1.0") }}
+
                         <div class="pure-form-message-inline">
                             {% if has_extra_headers_file %}
                                 <strong>Alert! Extra headers file found and will be added to this watch!</strong>

changedetectionio/tests/test_notification.py

@@ -431,15 +431,24 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
         follow_redirects=True
     )
 
-    ######### Test global/system settings - When everything is deleted it should give a helpful error
-    # See #2727
+    #2727 - be sure a test notification when there are zero watches works ( should all be deleted now)
+
+    os.unlink("test-datastore/notification.txt")
+
+
+    ######### Test global/system settings
     res = client.post(
         url_for("ajax_callback_send_notification_test")+"?mode=global-settings",
         data={"notification_urls": test_notification_url},
         follow_redirects=True
     )
-    assert res.status_code == 400
-    assert b"Error: You must have atleast one watch configured for 'test notification' to work" in res.data
 
+    assert res.status_code != 400
+    assert res.status_code != 500
 
+    # Give apprise time to fire
+    time.sleep(4)
 
+    with open("test-datastore/notification.txt", 'r') as f:
+        x = f.read()
+        assert 'change detection is cool 网站监测 内容更新了' in x

changedetectionio/tests/test_request.py

@@ -45,7 +45,7 @@ def test_headers_in_request(client, live_server, measure_memory_usage):
               "url": test_url,
               "tags": "",
               "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
-              "headers": "jinja2:{{ 1+1 }}\nxxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
+              "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -61,7 +61,6 @@ def test_headers_in_request(client, live_server, measure_memory_usage):
     )
 
     # Flask will convert the header key to uppercase
-    assert b"Jinja2:2" in res.data
     assert b"Xxx:ooo" in res.data
     assert b"Cool:yeah" in res.data
 
@@ -118,8 +117,7 @@ def test_body_in_request(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     # Now the change which should trigger a change
-    body_value = 'Test Body Value {{ 1+1 }}'
-    body_value_formatted = 'Test Body Value 2'
+    body_value = 'Test Body Value'
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
@@ -142,9 +140,8 @@ def test_body_in_request(client, live_server, measure_memory_usage):
 
     # If this gets stuck something is wrong, something should always be there
     assert b"No history found" not in res.data
-    # We should see the formatted value of what we sent in the reply
-    assert str.encode(body_value) not in res.data
-    assert str.encode(body_value_formatted) in res.data
+    # We should see what we sent in the reply
+    assert str.encode(body_value) in res.data
 
     ####### data sanity checks
     # Add the test URL twice, we will check

changedetectionio/tests/test_restock_itemprop.py

@@ -3,7 +3,7 @@ import os
 import time
 
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output, extract_UUID_from_client
+from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output
 from ..notification import default_notification_format
 
 instock_props = [
@@ -367,12 +367,6 @@ def test_change_with_notification_values(client, live_server):
         assert "new price 1950.45" in notification
         assert "title new price 1950.45" in notification
 
-    ## Now test the "SEND TEST NOTIFICATION" is working
-    os.unlink("test-datastore/notification.txt")
-    uuid = extract_UUID_from_client(client)
-    res = client.post(url_for("ajax_callback_send_notification_test", watch_uuid=uuid), data={}, follow_redirects=True)
-    time.sleep(5)
-    assert os.path.isfile("test-datastore/notification.txt"), "Notification received"
 
 
 def test_data_sanity(client, live_server):