Implement Jinja2 templating for notifications body and title.

Using Jinja2 templating actually simplifies slightly the code, but, most importantly, enables a much greater flexibility
to customize the notifications' content.

This change is breaking: the tokens like `{base_url}` should instead be written s `{{ base_url }}`. It would be very easy
to make it non-breaking, but I think it would be preferable to stick to the default Jinja2 syntax. To make it non-breaking,
the Jinja2 environment should be configured this way:
```
jinja2_env = Environment(
    loader=BaseLoader,
    variable_start_string='{',
    variable_end_string='}',
)
```

NB: This change could enable a few more follow-up enhancements:
* It could make sense as a later evolution to implement the same for the URL list. This could enable features like:
  - Customizing the sender based on the watch parameters through global settings
  - Sending the notification to a different email/channel/group based on a watch attribute, like its tag
* Jinja2 plays fine with complex variables. It would be relatively easy to expose many more variables that could be used
  in the notification.

Changes
=======
* Use Jinja2 template rendering to parse tokens in the notification body and title.
* Replace the settings validator to check that the body and title are both valid Jinja2 templates
  and that they do not contain any invalid token/variable.
* Update the corresponding documentation, setting pages and tests.

README.md

@@ -111,7 +111,7 @@ Just some examples
 
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications"  title="Self-hosted web page change monitoring notifications"  />
 
-Now you can also customise your notification content!
+Now you can also customise your notification content and use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2 templating</a> for their title and body!
 
 ### JSON API Monitoring
 

changedetectionio/forms.py

@@ -133,7 +133,7 @@ class ValidateAppRiseServers(object):
                 message = field.gettext('\'%s\' is not a valid AppRise URL.' % (server_url))
                 raise ValidationError(message)
 
-class ValidateTokensList(object):
+class ValidateJinja2Template(object):
     """
     Validates that a {token} is from a valid set
     """
@@ -142,11 +142,22 @@ class ValidateTokensList(object):
 
     def __call__(self, form, field):
         from changedetectionio import notification
-        regex = re.compile('{.*?}')
+        from jinja2 import Environment, BaseLoader, TemplateSyntaxError
-        for p in re.findall(regex, field.data):
+        from jinja2.meta import find_undeclared_variables
-            if not p.strip('{}') in notification.valid_tokens:
+
-                message = field.gettext('Token \'%s\' is not a valid token.')
+        try:
-                raise ValidationError(message % (p))
+            jinja2_env = Environment(loader=BaseLoader)
+            jinja2_env.globals.update(notification.valid_tokens)
+            rendered = jinja2_env.from_string(field.data).render()
+        except TemplateSyntaxError as e:
+            raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
+
+        ast = jinja2_env.parse(field.data)
+        undefined = ", ".join(find_undeclared_variables(ast))
+        if undefined:
+            raise ValidationError(
+                f"The following tokens used in the notification are not valid: {undefined}"
+            )
 
 class ValidateListRegex(object):
     """
@@ -201,8 +212,8 @@ class quickWatchForm(Form):
 class commonSettingsForm(Form):
 
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
-    notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {watch_url}', validators=[validators.Optional(), ValidateTokensList()])
+    notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
-    notification_body = TextAreaField('Notification Body', default='{watch_url} had a change.', validators=[validators.Optional(), ValidateTokensList()])
+    notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
     trigger_check = BooleanField('Send test notification on save')
     fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)

changedetectionio/notification.py

@@ -1,5 +1,5 @@
-import os
 import apprise
+from jinja2 import Environment, BaseLoader
 
 valid_tokens = {
     'base_url': '',
@@ -24,18 +24,13 @@ def process_notification(n_object, datastore):
         print (">> Process Notification: AppRise notifying {}".format(url))
         apobj.add(url)
 
-    # Get the notification body from datastore
-    n_body = n_object['notification_body']
-    n_title = n_object['notification_title']
-
     # Insert variables into the notification content
     notification_parameters = create_notification_parameters(n_object, datastore)
 
-    for n_k in notification_parameters:
+    # Get the notification body from datastore
-        token = '{' + n_k + '}'
+    jinja2_env = Environment(loader=BaseLoader)
-        val = notification_parameters[n_k]
+    n_body = jinja2_env.from_string(n_object['notification_body']).render(**notification_parameters)
-        n_title = n_title.replace(token, val)
+    n_title = jinja2_env.from_string(n_object['notification_title']).render(**notification_parameters)
-        n_body = n_body.replace(token, val)
 
     apobj.notify(
         body=n_body,
@@ -61,7 +56,7 @@ def create_notification_parameters(n_object, datastore):
 
     watch_url = n_object['watch_url']
 
-    # Re #148 - Some people have just {base_url} in the body or title, but this may break some notification services
+    # Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services
     #           like 'Join', so it's always best to atleast set something obvious so that they are not broken.
     if base_url == '':
         base_url = "<base-url-env-var-not-set>"

changedetectionio/templates/_common_fields.jinja

@@ -26,8 +26,10 @@
                             </div>
                             <div class="pure-controls">
                             <span class="pure-form-message-inline">
-                                These tokens can be used in the notification body and title to
+                                You can use
-                                customise the notification text.
+                                <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a>
+                                templating for the notification title and body.<br />
+                                These tokens can be used to customise them.
                             </span>
                                 <table class="pure-table" id="token-table">
                                     <thead>
@@ -38,42 +40,42 @@
                                     </thead>
                                     <tbody>
                                     <tr>
-                                        <td><code>{base_url}</code></td>
+                                        <td><code>{{ '{{ base_url }}' }}</code></td>
                                         <td>The URL of the changedetection.io instance you are running.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{watch_url}</code></td>
+                                        <td><code>{{ '{{ watch_url }}' }}</code></td>
                                         <td>The URL being watched.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{watch_uuid}</code></td>
+                                        <td><code>{{ '{{ watch_uuid }}' }}</code></td>
                                         <td>The UUID of the watch.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{watch_title}</code></td>
+                                        <td><code>{{ '{{ watch_title }}' }}</code></td>
                                         <td>The title of the watch.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{watch_tag}</code></td>
+                                        <td><code>{{ '{{ watch_tag }}' }}</code></td>
                                         <td>The tag of the watch.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{preview_url}</code></td>
+                                        <td><code>{{ '{{ preview_url }}' }}</code></td>
                                         <td>The URL of the preview page generated by changedetection.io.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{diff_url}</code></td>
+                                        <td><code>{{ '{{ diff_url }}' }}</code></td>
                                         <td>The URL of the diff page generated by changedetection.io.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{current_snapshot}</code></td>
+                                        <td><code>{{ '{{ current_snapshot }}' }}</code></td>
                                         <td>The current snapshot value, useful when combined with JSON or CSS filters
                                         </td>
                                     </tr>
                                     </tbody>
                                 </table>
                                 <span class="pure-form-message-inline">
-                                URLs generated by changedetection.io (such as <code>{diff_url}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
+                                URLs generated by changedetection.io (such as <code>{{ '{{ diff_url }}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
                                 Your <code>BASE_URL</code> var is currently "{{base_url}}"
                             </span>
                             </div>

changedetectionio/templates/settings.html

@@ -36,7 +36,7 @@
                         {{ render_field(form.base_url, placeholder="http://yoursite.com:5000/",
                         class="m-d") }}
                         <span class="pure-form-message-inline">
-                            Base URL used for the {base_url} token in notifications, default value is the ENV var 'base_url',
+                            Base URL used for the {{ '{{ base_url }}' }} token in notifications, default value is the ENV var 'base_url',
                             <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
                         </span>
                     </div>

changedetectionio/tests/test_notification.py

@@ -46,15 +46,15 @@ def test_check_notification(client, live_server):
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"notification_urls": notification_url,
-              "notification_title": "New ChangeDetection.io Notification - {watch_url}",
+              "notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
-              "notification_body": "BASE URL: {base_url}\n"
+              "notification_body": "BASE URL: {{ base_url }}\n"
-                                   "Watch URL: {watch_url}\n"
+                                   "Watch URL: {{ watch_url }}\n"
-                                   "Watch UUID: {watch_uuid}\n"
+                                   "Watch UUID: {{ watch_uuid }}\n"
-                                   "Watch title: {watch_title}\n"
+                                   "Watch title: {{ watch_title }}\n"
-                                   "Watch tag: {watch_tag}\n"
+                                   "Watch tag: {{ watch_tag }}\n"
-                                   "Preview: {preview_url}\n"
+                                   "Preview: {{ preview_url }}\n"
-                                   "Diff URL: {diff_url}\n"
+                                   "Diff URL: {{ diff_url }}\n"
-                                   "Snapshot: {current_snapshot}\n"
+                                   "Snapshot: {{ current_snapshot }}\n"
                                    ":-)",
               "url": test_url,
               "tag": "my tag",
@@ -126,7 +126,7 @@ def test_check_notification(client, live_server):
 
     res = client.post(
         url_for("settings_page"),
-        data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
+        data={"notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
               "notification_urls": "json://foobar.com", #Re #143 should not see that it sent without [test checkbox]
               "minutes_between_check": 180,
               "fetch_backend": "html_requests",
@@ -179,13 +179,27 @@ def test_check_notification(client, live_server):
     # Now adding a wrong token should give us an error
     res = client.post(
         url_for("settings_page"),
-        data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
+        data={"notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
-              "notification_body": "Rubbish: {rubbish}\n",
+              "notification_body": "Rubbish: {{ rubbish }}\n",
               "notification_urls": "json://foobar.com",
               "minutes_between_check": 180,
               "fetch_backend": "html_requests"
               },
         follow_redirects=True
     )
+    assert bytes("The following tokens used in the notification are not valid".encode('utf-8')) in res.data
+
+
+    # And trying to define an invalid Jinja2 template should also throw an error
+    res = client.post(
+        url_for("settings_page"),
+        data={"notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
+              "notification_body": "Rubbish: {{ rubbish }\n",
+              "notification_urls": "json://foobar.com",
+              "minutes_between_check": 180,
+              "fetch_backend": "html_requests"
+              },
+        follow_redirects=True
+    )
+    assert bytes("This is not a valid Jinja2 template".encode('utf-8')) in res.data
 
-    assert bytes("is not a valid token".encode('utf-8')) in res.data

requirements.txt

@@ -16,8 +16,9 @@ chardet > 2.3.0
 wtforms ~= 2.3.3
 jsonpath-ng ~= 1.5.3
 
-# Notification library
+# Notification libraries
 apprise ~= 0.9
+jinja2 ~= 2.11.3
 
 # Pinned version of cryptography otherwise
 # ERROR: Could not build wheels for cryptography which use PEP 517 and cannot be installed directly