Closes #2286

.github/dependabot.yml

@@ -4,10 +4,6 @@ updates:
     directory: /
     schedule:
       interval: "weekly"
-    "caronc/apprise":
-      versioning-strategy: "increase"
-      schedule:
-        interval: "daily"
     groups:
       all:
         patterns:

.github/workflows/test-only.yml

@@ -59,7 +59,6 @@ jobs:
           echo "run test with unittest"
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
           
           # All tests
           echo "run test with pytest"

changedetectionio/__init__.py

@@ -2,12 +2,12 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.45.21'
+__version__ = '0.45.17'
 
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 from json.decoder import JSONDecodeError
 import os
-os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
+#os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
 import eventlet
 import eventlet.wsgi
 import getopt

changedetectionio/api/api_v1.py

@@ -1,5 +1,5 @@
 import os
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 
 from flask_expects_json import expects_json
 from changedetectionio import queuedWatchMetaData

changedetectionio/blueprint/browser_steps/__init__.py

@@ -12,7 +12,7 @@
 #
 #
 
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 from flask import Blueprint, request, make_response
 import os
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -7,7 +7,6 @@ from random import randint
 from loguru import logger
 
 from changedetectionio.content_fetchers.base import manage_user_agent
-from changedetectionio.safe_jinja import render as jinja_render
 
 # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
 # 0- off, 1- on
@@ -65,12 +64,14 @@ class steppable_browser_interface():
         action_handler = getattr(self, "action_" + call_action_name)
 
         # Support for Jinja2 variables in the value and selector
+        from jinja2 import Environment
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
 
         if selector and ('{%' in selector or '{{' in selector):
-            selector = jinja_render(template_str=selector)
+            selector = str(jinja2_env.from_string(selector).render())
 
         if optional_value and ('{%' in optional_value or '{{' in optional_value):
-            optional_value = jinja_render(template_str=optional_value)
+            optional_value = str(jinja2_env.from_string(optional_value).render())
 
         action_handler(selector, optional_value)
         self.page.wait_for_timeout(1.5 * 1000)

changedetectionio/blueprint/check_proxies/__init__.py

@@ -31,9 +31,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         import time
         from changedetectionio.content_fetchers import exceptions as content_fetcher_exceptions
         from changedetectionio.processors import text_json_diff
-        from changedetectionio.safe_jinja import render as jinja_render
 
         status = {'status': '', 'length': 0, 'text': ''}
+        from jinja2 import Environment, BaseLoader
 
         contents = ''
         now = time.time()
@@ -64,9 +64,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             status.update({'status': 'OK', 'length': len(contents), 'text': ''})
 
         if status.get('text'):
-            # parse 'text' as text for safety
-            v = {'text': status['text']}
-            status['text'] = jinja_render(template_str='{{text|e}}', **v)
+            status['text'] = Environment(loader=BaseLoader()).from_string('{{text|e}}').render({'text': status['text']})
 
         status['time'] = "{:.2f}s".format(time.time() - now)
 

changedetectionio/blueprint/price_data_follower/__init__.py

@@ -1,5 +1,5 @@
 
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 from flask import Blueprint, flash, redirect, url_for
 from flask_login import login_required
 from changedetectionio.store import ChangeDetectionStore

changedetectionio/blueprint/tags/__init__.py

@@ -12,15 +12,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         from .form import SingleTag
         add_form = SingleTag(request.form)
         sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
-
-        from collections import Counter
-
-        tag_count = Counter(tag for watch in datastore.data['watching'].values() if watch.get('tags') for tag in watch['tags'])
-
         output = render_template("groups-overview.html",
-                                 available_tags=sorted_tags,
                                  form=add_form,
-                                 tag_count=tag_count
+                                 available_tags=sorted_tags,
                                  )
 
         return output

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -3,7 +3,7 @@
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test', mode="group-settings")}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -27,7 +27,6 @@
             <thead>
             <tr>
                 <th></th>
-                <th># Watches</th>
                 <th>Tag / Label name</th>
                 <th></th>
             </tr>
@@ -46,8 +45,7 @@
                 <td class="watch-controls">
                     <a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
                 </td>
-                <td>{{ "{:,}".format(tag_count[uuid]) if uuid in tag_count else 0 }}</td>
-                <td class="title-col inline"> <a href="{{url_for('index', tag=uuid) }}">{{ tag.title }}</a></td>
+                <td class="title-col inline">{{tag.title}}</td>
                 <td>
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">Edit</a>&nbsp;
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.delete', uuid=uuid) }}" title="Deletes and removes tag">Delete</a>

changedetectionio/content_fetchers/__init__.py

@@ -1,5 +1,5 @@
 import sys
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 from loguru import logger
 from changedetectionio.content_fetchers.exceptions import BrowserStepsStepException
 import os

changedetectionio/content_fetchers/base.py

@@ -123,7 +123,8 @@ class Fetcher():
     def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
         from playwright._impl._errors import TimeoutError, Error
-        from changedetectionio.safe_jinja import render as jinja_render
+        from jinja2 import Environment
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
 
         step_n = 0
 
@@ -142,9 +143,9 @@ class Fetcher():
                     selector = step['selector']
                     # Support for jinja2 template in step values, with date module added
                     if '{%' in step['optional_value'] or '{{' in step['optional_value']:
-                        optional_value = jinja_render(template_str=step['optional_value'])
+                        optional_value = str(jinja2_env.from_string(step['optional_value']).render())
                     if '{%' in step['selector'] or '{{' in step['selector']:
-                        selector = jinja_render(template_str=step['selector'])
+                        selector = str(jinja2_env.from_string(step['selector']).render())
 
                     getattr(interface, "call_action")(action_name=step['operation'],
                                                       selector=selector,

changedetectionio/flask_app.py

@@ -5,11 +5,11 @@ import os
 import queue
 import threading
 import time
-from .safe_jinja import render as jinja_render
-from changedetectionio.strtobool import strtobool
 from copy import deepcopy
+from distutils.util import strtobool
 from functools import wraps
 from threading import Event
+
 import flask_login
 import pytz
 import timeago
@@ -319,6 +319,8 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/rss", methods=['GET'])
     def rss():
+        from jinja2 import Environment, BaseLoader
+        jinja2_env = Environment(loader=BaseLoader)
         now = time.time()
         # Always requires token set
         app_rss_token = datastore.data['settings']['application'].get('rss_access_token')
@@ -386,7 +388,7 @@ def changedetection_app(config=None, datastore_o=None):
                 # @todo Make this configurable and also consider html-colored markup
                 # @todo User could decide if <link> goes to the diff page, or to the watch link
                 rss_template = "<html><body>\n<h4><a href=\"{{watch_url}}\">{{watch_title}}</a></h4>\n<p>{{html_diff}}</p>\n</body></html>\n"
-                content = jinja_render(template_str=rss_template, watch_title=watch_title, html_diff=html_diff, watch_url=watch.link)
+                content = jinja2_env.from_string(rss_template).render(watch_title=watch_title, html_diff=html_diff, watch_url=watch.link)
 
                 fe.content(content=content, type='CDATA')
 
@@ -517,29 +519,26 @@ def changedetection_app(config=None, datastore_o=None):
         notification_urls = request.form['notification_urls'].strip().splitlines()
 
         if not notification_urls:
-            logger.debug("Test notification - Trying by group/tag in the edit form if available")
-            # On an edit page, we should also fire off to the tags if they have notifications
-            if request.form.get('tags') and request.form['tags'].strip():
+            logger.debug("Test notification - Trying by group/tag")
+            if request.form['tags'].strip():
                 for k in request.form['tags'].split(','):
                     tag = datastore.tag_exists_by_name(k.strip())
                     notification_urls = tag.get('notifications_urls') if tag and tag.get('notifications_urls') else None
 
-        is_global_settings_form = request.args.get('mode', '') == 'global-settings'
-        is_group_settings_form = request.args.get('mode', '') == 'group-settings'
-        if not notification_urls and not is_global_settings_form and not is_group_settings_form:
-            # In the global settings, use only what is typed currently in the text box
+        if not notification_urls:
             logger.debug("Test notification - Trying by global system settings notifications")
             if datastore.data['settings']['application'].get('notification_urls'):
                 notification_urls = datastore.data['settings']['application']['notification_urls']
 
 
         if not notification_urls:
-            return 'No Notification URLs set/found'
+            return make_response({'error': 'No Notification URLs set'}, 400)
 
         for n_url in notification_urls:
             if len(n_url.strip()):
                 if not apobj.add(n_url):
-                    return f'Error - {n_url} is not a valid AppRise URL.'
+                    message = '{} is not a valid AppRise URL.'.format(n_url)
+                    return make_response({'error': message}, 400)
 
         try:
             # use the same as when it is triggered, but then override it with the form test values
@@ -564,7 +563,7 @@ def changedetection_app(config=None, datastore_o=None):
         except Exception as e:
             return make_response({'error': str(e)}, 400)
 
-        return 'OK - Sent test notifications'
+        return 'OK'
 
 
     @app.route("/clear_history/<string:uuid>", methods=['GET'])
@@ -601,12 +600,6 @@ def changedetection_app(config=None, datastore_o=None):
         output = render_template("clear_all_history.html")
         return output
 
-    def _watch_has_tag_options_set(watch):
-        """This should be fixed better so that Tag is some proper Model, a tag is just a Watch also"""
-        for tag_uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
-            if tag_uuid in watch.get('tags', []) and (tag.get('include_filters') or tag.get('subtractive_selectors')):
-                return True
-
     @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
     @login_optionally_required
     # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists
@@ -777,7 +770,6 @@ def changedetection_app(config=None, datastore_o=None):
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
                                      has_empty_checktime=using_default_check_time,
                                      has_extra_headers_file=len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
-                                     has_special_tag_options=_watch_has_tag_options_set(watch=watch),
                                      is_html_webdriver=is_html_webdriver,
                                      jq_support=jq_support,
                                      playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),

changedetectionio/forms.py

@@ -1,6 +1,6 @@
 import os
 import re
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 
 from wtforms import (
     BooleanField,
@@ -236,26 +236,21 @@ class ValidateJinja2Template(object):
     def __call__(self, form, field):
         from changedetectionio import notification
 
-        from jinja2 import BaseLoader, TemplateSyntaxError, UndefinedError
-        from jinja2.sandbox import ImmutableSandboxedEnvironment
+        from jinja2 import Environment, BaseLoader, TemplateSyntaxError, UndefinedError
         from jinja2.meta import find_undeclared_variables
-        import jinja2.exceptions
 
-        # Might be a list of text, or might be just text (like from the apprise url list)
-        joined_data = ' '.join(map(str, field.data)) if isinstance(field.data, list) else f"{field.data}"
 
         try:
-            jinja2_env = ImmutableSandboxedEnvironment(loader=BaseLoader)
+            jinja2_env = Environment(loader=BaseLoader)
             jinja2_env.globals.update(notification.valid_tokens)
-            jinja2_env.from_string(joined_data).render()
+
+            rendered = jinja2_env.from_string(field.data).render()
         except TemplateSyntaxError as e:
             raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
         except UndefinedError as e:
             raise ValidationError(f"A variable or function is not defined: {e}") from e
-        except jinja2.exceptions.SecurityError as e:
-            raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
 
-        ast = jinja2_env.parse(joined_data)
+        ast = jinja2_env.parse(field.data)
         undefined = ", ".join(find_undeclared_variables(ast))
         if undefined:
             raise ValidationError(
@@ -420,7 +415,7 @@ class quickWatchForm(Form):
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
 
-    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
+    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_format = SelectField('Notification format', choices=valid_notification_formats.keys())
@@ -504,9 +499,11 @@ class watchForm(commonSettingsForm):
             result = False
 
         # Attempt to validate jinja2 templates in the URL
+        from jinja2 import Environment
+        # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
         try:
-            from changedetectionio.safe_jinja import render as jinja_render
-            jinja_render(template_str=self.url.data)
+            ready_url = str(jinja2_env.from_string(self.url.data).render())
         except Exception as e:
             self.url.errors.append('Invalid template syntax')
             result = False

changedetectionio/html_tools.py

@@ -169,14 +169,14 @@ def xpath1_filter(xpath_filter, html_content, append_pretty_line_formatting=Fals
         # And where the matched result doesn't include something that will cause Inscriptis to add a newline
         # (This way each 'match' reliably has a new-line in the diff)
         # Divs are converted to 4 whitespaces by inscriptis
-        if append_pretty_line_formatting and len(html_block) and (not hasattr(element, 'tag') or not element.tag in (['br', 'hr', 'div', 'p'])):
+        if append_pretty_line_formatting and len(html_block) and (not hasattr( element, 'tag' ) or not element.tag in (['br', 'hr', 'div', 'p'])):
             html_block += TEXT_FILTER_LIST_LINE_SUFFIX
 
-        # Some kind of text, UTF-8 or other
-        if isinstance(element, (str, bytes)):
-            html_block += element
+        if type(element) == etree._ElementStringResult:
+            html_block += str(element)
+        elif type(element) == etree._ElementUnicodeResult:
+            html_block += str(element)
         else:
-            # Return the HTML which will get parsed as text
             html_block += etree.tostring(element, pretty_print=True).decode('utf-8')
 
     return html_block

changedetectionio/model/Watch.py

@@ -1,6 +1,4 @@
-from changedetectionio.strtobool import strtobool
-from changedetectionio.safe_jinja import render as jinja_render
-
+from distutils.util import strtobool
 import os
 import re
 import time
@@ -139,11 +137,12 @@ class model(dict):
 
         ready_url = url
         if '{%' in url or '{{' in url:
+            from jinja2 import Environment
             # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
+            jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
             try:
-                ready_url = jinja_render(template_str=url)
+                ready_url = str(jinja2_env.from_string(url).render())
             except Exception as e:
-                logger.critical(f"Invalid URL template for: '{url}' - {str(e)}")
                 from flask import (
                     flash, Markup, url_for
                 )
@@ -363,7 +362,6 @@ class model(dict):
         # @todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status
         return snapshot_fname
 
-    @property
     @property
     def has_empty_checktime(self):
         # using all() + dictionary comprehension

changedetectionio/notification.py

@@ -1,5 +1,6 @@
 import apprise
 import time
+from jinja2 import Environment, BaseLoader
 from apprise import NotifyFormat
 import json
 from loguru import logger
@@ -115,7 +116,6 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
 
 def process_notification(n_object, datastore):
 
-    from .safe_jinja import render as jinja_render
     now = time.time()
     if n_object.get('notification_timestamp'):
         logger.trace(f"Time since queued {now-n_object['notification_timestamp']:.3f}s")
@@ -123,9 +123,9 @@ def process_notification(n_object, datastore):
     notification_parameters = create_notification_parameters(n_object, datastore)
 
     # Get the notification body from datastore
-    n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
-    n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
-
+    jinja2_env = Environment(loader=BaseLoader)
+    n_body = jinja2_env.from_string(n_object.get('notification_body', '')).render(**notification_parameters)
+    n_title = jinja2_env.from_string(n_object.get('notification_title', '')).render(**notification_parameters)
     n_format = valid_notification_formats.get(
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
@@ -157,7 +157,7 @@ def process_notification(n_object, datastore):
                 continue
 
             logger.info(">> Process Notification: AppRise notifying {}".format(url))
-            url = jinja_render(template_str=url, **notification_parameters)
+            url = jinja2_env.from_string(url).render(**notification_parameters)
 
             # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
             # Because different notifications may require different pre-processing, run each sequentially :(

changedetectionio/processors/__init__.py

@@ -3,7 +3,7 @@ import os
 import hashlib
 import re
 from copy import deepcopy
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 from loguru import logger
 
 class difference_detection_processor():

changedetectionio/safe_jinja.py

@@ -1,18 +0,0 @@
-"""
-Safe Jinja2 render with max payload sizes
-
-See https://jinja.palletsprojects.com/en/3.1.x/sandbox/#security-considerations
-"""
-
-import jinja2.sandbox
-import typing as t
-import os
-
-JINJA2_MAX_RETURN_PAYLOAD_SIZE = 1024 * int(os.getenv("JINJA2_MAX_RETURN_PAYLOAD_SIZE_KB", 1024 * 10))
-
-
-def render(template_str, **args: t.Any) -> str:
-    jinja2_env = jinja2.sandbox.ImmutableSandboxedEnvironment(extensions=['jinja2_time.TimeExtension'])
-    output = jinja2_env.from_string(template_str).render(args)
-    return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]
-

changedetectionio/static/js/notifications.js

@@ -45,7 +45,7 @@ $(document).ready(function() {
       }
     }).done(function(data){
       console.log(data);
-      alert(data);
+      alert('Sent');
     }).fail(function(data){
       console.log(data);
       alert('There was an error communicating with the server.');

changedetectionio/static/styles/diff.css

@@ -68,7 +68,7 @@
   --color-last-checked: #bbb;
   --color-text-footer: #444;
   --color-border-watch-table-cell: #eee;
-  --color-text-watch-tag-list: rgba(231, 0, 105, 0.4);
+  --color-text-watch-tag-list: #e70069;
   --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
   --color-background-new-watch-input: var(--color-white);
   --color-text-new-watch-input: var(--color-text);
@@ -111,7 +111,7 @@ html[data-darkmode="true"] {
   --color-background-input: var(--color-grey-350);
   --color-text-input-description: var(--color-grey-600);
   --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: rgba(250, 62, 146, 0.4);
+  --color-text-watch-tag-list: #fa3e92;
   --color-background-code: var(--color-grey-200);
   --color-background-tab: rgba(0, 0, 0, 0.2);
   --color-background-tab-hover: rgba(0, 0, 0, 0.5);

changedetectionio/static/styles/scss/parts/_variables.scss

@@ -75,7 +75,7 @@
   --color-text-footer: #444;
   --color-border-watch-table-cell: #eee;
 
-  --color-text-watch-tag-list: rgba(231, 0, 105, 0.4);
+  --color-text-watch-tag-list: #e70069;
   --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
   --color-background-new-watch-input: var(--color-white);
   --color-text-new-watch-input: var(--color-text);
@@ -127,7 +127,7 @@ html[data-darkmode="true"] {
   --color-background-input: var(--color-grey-350);
   --color-text-input-description: var(--color-grey-600);
   --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: rgba(250, 62, 146, 0.4);
+  --color-text-watch-tag-list: #fa3e92;
   --color-background-code: var(--color-grey-200);
 
   --color-background-tab: rgba(0, 0, 0, 0.2);

changedetectionio/static/styles/scss/styles.scss

@@ -187,11 +187,8 @@ code {
 }
 
 .watch-tag-list {
-  color: var(--color-white);
+  color: var(--color-text-watch-tag-list);
   white-space: nowrap;
-  background: var(--color-text-watch-tag-list);
-  border-radius: 5px;
-  padding: 2px 5px;
 }
 
 .box {

changedetectionio/static/styles/styles.css

@@ -284,7 +284,7 @@ ul#requests-extra_browsers {
   --color-last-checked: #bbb;
   --color-text-footer: #444;
   --color-border-watch-table-cell: #eee;
-  --color-text-watch-tag-list: rgba(231, 0, 105, 0.4);
+  --color-text-watch-tag-list: #e70069;
   --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
   --color-background-new-watch-input: var(--color-white);
   --color-text-new-watch-input: var(--color-text);
@@ -327,7 +327,7 @@ html[data-darkmode="true"] {
   --color-background-input: var(--color-grey-350);
   --color-text-input-description: var(--color-grey-600);
   --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: rgba(250, 62, 146, 0.4);
+  --color-text-watch-tag-list: #fa3e92;
   --color-background-code: var(--color-grey-200);
   --color-background-tab: rgba(0, 0, 0, 0.2);
   --color-background-tab-hover: rgba(0, 0, 0, 0.5);
@@ -532,11 +532,8 @@ code {
     margin: 0 3px 0 5px; }
 
 .watch-tag-list {
-  color: var(--color-white);
-  white-space: nowrap;
-  background: var(--color-text-watch-tag-list);
-  border-radius: 5px;
-  padding: 2px 5px; }
+  color: var(--color-text-watch-tag-list);
+  white-space: nowrap; }
 
 .box {
   max-width: 80%;

changedetectionio/store.py

@@ -1,4 +1,4 @@
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 
 from flask import (
     flash

changedetectionio/strtobool.py

@@ -1,23 +0,0 @@
-# Because strtobool was removed in python 3.12 distutils
-
-_MAP = {
-    'y': True,
-    'yes': True,
-    't': True,
-    'true': True,
-    'on': True,
-    '1': True,
-    'n': False,
-    'no': False,
-    'f': False,
-    'false': False,
-    'off': False,
-    '0': False
-}
-
-
-def strtobool(value):
-    try:
-        return _MAP[str(value).lower()]
-    except KeyError:
-        raise ValueError('"{}" is not a valid bool value'.format(value))

changedetectionio/templates/edit.html

@@ -7,8 +7,7 @@
 <script>
     const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
-    <!-- Should be _external so that firefox and others load it more reliably -->
-    const browser_steps_fetch_screenshot_image_url="{{url_for('browser_steps.browser_steps_fetch_screenshot_image', uuid=uuid, _external=True)}}";
+    const browser_steps_fetch_screenshot_image_url="{{url_for('browser_steps.browser_steps_fetch_screenshot_image', uuid=uuid)}}";
     const browser_steps_last_error_step={{ watch.browser_steps_last_error_step|tojson }};
     const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
     const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
@@ -32,7 +31,6 @@
 <script src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
 {% endif %}
 
-{% set has_tag_filters_extra="WARNING: Watch has tag/groups set with special filters\n" if has_special_tag_options else '' %}
 <script src="{{url_for('static_content', group='js', filename='recheck-proxy.js')}}" defer></script>
 
 <div class="edit-form monospaced-textarea">
@@ -282,7 +280,7 @@ User-Agent: wonderbra 1.0") }}
                     <div class="pure-control-group">
                         {% set field = render_field(form.include_filters,
                             rows=5,
-                            placeholder=has_tag_filters_extra+"#example
+                            placeholder="#example
 xpath://body/div/span[contains(@class, 'example-class')]",
                             class="m-d")
                         %}
@@ -318,7 +316,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                 </span>
                     </div>
                 <fieldset class="pure-control-group">
-                    {{ render_field(form.subtractive_selectors, rows=5, placeholder=has_tag_filters_extra+"header
+                    {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
 footer
 nav
 .stockticker") }}

changedetectionio/templates/settings.html

@@ -4,7 +4,7 @@
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test', mode="global-settings")}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
 {% endif %}

changedetectionio/templates/watch-overview.html

@@ -169,7 +169,7 @@
                 <td>
                     <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                        class="recheck pure-button pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
-                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
+                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button pure-button-primary">Edit</a>
                     {% if watch.history_n >= 2 %}
 
                         {%  if is_unviewed %}

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python3
-import os.path
+
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
@@ -107,6 +107,7 @@ def test_check_add_line_contains_trigger(client, live_server):
     #live_server_setup(live_server)
 
     # Give the endpoint time to spin up
+    time.sleep(1)
     test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://') + "?xxx={{ watch_url }}"
 
     res = client.post(
@@ -165,7 +166,6 @@ def test_check_add_line_contains_trigger(client, live_server):
 
     # Takes a moment for apprise to fire
     time.sleep(3)
-    assert os.path.isfile("test-datastore/notification.txt"), "Notification fired because I can see the output file"
     with open("test-datastore/notification.txt", 'r') as f:
         response= f.read()
         assert '-Oh yes please-' in response

changedetectionio/tests/test_group.py

@@ -100,12 +100,6 @@ def test_setup_group_tag(client, live_server):
     assert b'Should be only this' in res.data
     assert b'And never this' not in res.data
 
-    res = client.get(
-        url_for("edit_page", uuid="first"),
-        follow_redirects=True
-    )
-    # 2307 the UI notice should appear in the placeholder
-    assert b'WARNING: Watch has tag/groups set with special filters' in res.data
 
     # RSS Group tag filter
     # An extra one that should be excluded

changedetectionio/tests/test_jinja2.py

@@ -2,15 +2,15 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from .util import live_server_setup
 
 
-def test_setup(client, live_server):
-    live_server_setup(live_server)
-
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
 def test_jinja2_in_url_query(client, live_server):
-    #live_server_setup(live_server)
+    live_server_setup(live_server)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_return_query', _external=True)
@@ -24,35 +24,10 @@ def test_jinja2_in_url_query(client, live_server):
         follow_redirects=True
     )
     assert b"Watch added" in res.data
-    wait_for_all_checks(client)
-
+    time.sleep(3)
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True
     )
     assert b'date=2' in res.data
-
-# https://techtonics.medium.com/secure-templating-with-jinja2-understanding-ssti-and-jinja2-sandbox-environment-b956edd60456
-def test_jinja2_security_url_query(client, live_server):
-    #live_server_setup(live_server)
-
-    # Add our URL to the import page
-    test_url = url_for('test_return_query', _external=True)
-
-    # because url_for() will URL-encode the var, but we dont here
-    full_url = "{}?{}".format(test_url,
-                              "date={{ ''.__class__.__mro__[1].__subclasses__()}}", )
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": full_url, "tags": "test"},
-        follow_redirects=True
-    )
-    assert b"Watch added" in res.data
-    wait_for_all_checks(client)
-
-    # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
-    assert b'is invalid and cannot be used' in res.data
-    # Some of the spewed output from the subclasses
-    assert b'dict_values' not in res.data

changedetectionio/tests/test_xpath_selector.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+#!/usr/bin/python3
 
 import time
 from flask import url_for
@@ -255,69 +255,6 @@ def test_xpath23_prefix_validation(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_xpath1_lxml(client, live_server):
-    #live_server_setup(live_server)
-
-    d = '''<?xml version="1.0" encoding="UTF-8"?>
-    <rss xmlns:taxo="http://purl.org/rss/1.0/modules/taxonomy/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
-    	<channel>
-    		<title>rpilocator.com</title>
-    		<link>https://rpilocator.com</link>
-    		<description>Find Raspberry Pi Computers in Stock</description>
-    		<lastBuildDate>Thu, 19 May 2022 23:27:30 GMT</lastBuildDate>
-    		<image>
-    			<url>https://rpilocator.com/favicon.png</url>
-    			<title>rpilocator.com</title>
-    			<link>https://rpilocator.com/</link>
-    			<width>32</width>
-    			<height>32</height>
-    		</image>
-    		<item>
-    			<title>Stock Alert (UK): RPi CM4</title>
-    			<foo>something else unrelated</foo>
-    		</item>
-    		<item>
-    			<title>Stock Alert (UK): Big monitorěěěě</title>
-    			<foo>something else unrelated</foo>
-    		</item>		
-    	</channel>
-    </rss>'''.encode('utf-8')
-
-    with open("test-datastore/endpoint-content.txt", "wb") as f:
-        f.write(d)
-
-
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"include_filters": "xpath1://title/text()", "url": test_url, "tags": "", "headers": "",
-              'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    ##### #2312
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-    assert b'_ElementStringResult' not in res.data # tested with 5.1.1 when it was removed and 5.1.0
-    assert b'Exception' not in res.data
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert b"rpilocator.com" in res.data  # in selector
-    assert "Stock Alert (UK): Big monitorěěěě".encode('utf-8') in res.data  # not in selector
-
-    #####
-
 
 def test_xpath1_validation(client, live_server):
     # Add our URL to the import page

changedetectionio/tests/unit/test_jinja2_security.py

@@ -1,57 +0,0 @@
-#!/usr/bin/python3
-
-# run from dir above changedetectionio/ dir
-# python3 -m unittest changedetectionio.tests.unit.test_jinja2_security
-
-import unittest
-from changedetectionio import safe_jinja
-
-
-# mostly
-class TestJinja2SSTI(unittest.TestCase):
-
-    def test_exception(self):
-        import jinja2
-
-        # Where sandbox should kick in
-        attempt_list = [
-            "My name is {{ self.__init__.__globals__.__builtins__.__import__('os').system('id') }}",
-            "{{ self._TemplateReference__context.cycler.__init__.__globals__.os }}",
-            "{{ self.__init__.__globals__.__builtins__.__import__('os').popen('id').read() }}",
-            "{{cycler.__init__.__globals__.os.popen('id').read()}}",
-            "{{joiner.__init__.__globals__.os.popen('id').read()}}",
-            "{{namespace.__init__.__globals__.os.popen('id').read()}}",
-            "{{ ''.__class__.__mro__[2].__subclasses__()[40]('/tmp/hello.txt', 'w').write('Hello here !') }}",
-            "My name is {{ self.__init__.__globals__ }}",
-            "{{ dict.__base__.__subclasses__() }}"
-        ]
-        for attempt in attempt_list:
-            with self.assertRaises(jinja2.exceptions.SecurityError):
-                safe_jinja.render(attempt)
-
-    def test_exception_debug_calls(self):
-        import jinja2
-        # Where sandbox should kick in - configs and debug calls
-        attempt_list = [
-            "{% debug %}",
-        ]
-        for attempt in attempt_list:
-            # Usually should be something like 'Encountered unknown tag 'debug'.'
-            with self.assertRaises(jinja2.exceptions.TemplateSyntaxError):
-                safe_jinja.render(attempt)
-
-    # https://book.hacktricks.xyz/pentesting-web/ssti-server-side-template-injection/jinja2-ssti#accessing-global-objects
-    def test_exception_empty_calls(self):
-        import jinja2
-        attempt_list = [
-            "{{config}}",
-            "{{ debug }}"
-            "{{[].__class__}}",
-        ]
-        for attempt in attempt_list:
-            self.assertEqual(len(safe_jinja.render(attempt)), 0, f"string test '{attempt}' is correctly empty")
-
-
-
-if __name__ == '__main__':
-    unittest.main()

changedetectionio/tests/util.py

@@ -116,7 +116,7 @@ def extract_UUID_from_client(client):
     )
     # <span id="api-key">{{api_key}}</span>
 
-    m = re.search('edit/(.+?)[#"]', str(res.data))
+    m = re.search('edit/(.+?)"', str(res.data))
     uuid = m.group(1)
     return uuid.strip()
 

changedetectionio/update_worker.py

@@ -462,7 +462,7 @@ class update_worker(threading.Thread):
                     except Exception as e:
                         logger.error(f"Exception reached processing watch UUID: {uuid}")
                         logger.error(str(e))
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Exception: " + str(e)})
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
                         # Other serious error
                         process_changedetection_results = False
 #                        import traceback

requirements.txt

@@ -52,7 +52,7 @@ cryptography~=3.4
 beautifulsoup4
 
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
-lxml >=4.8.0,<6
+lxml
 
 # XPath 2.0-3.1 support - 4.2.0 broke something?
 elementpath==4.1.5