error text should contain the word Exception

.github/workflows/test-only.yml

@@ -59,7 +59,6 @@ jobs:
           echo "run test with unittest"
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
           
           # All tests
           echo "run test with pytest"

README.md

@@ -257,7 +257,13 @@ Supports managing the website watch list [via our API](https://changedetection.i
 Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
 
 
-Consider taking out an officially supported [website change detection subscription](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
+Firstly, consider taking out an officially supported [website change detection subscription](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
+
+Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ)
+
+Or BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/btc-support.png" style="max-width:50%;" alt="Support us!"  />
 
 ## Commercial Support
 

changedetectionio/__init__.py

@@ -2,12 +2,12 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.45.22'
+__version__ = '0.45.20'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
 import os
-os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
+#os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
 import eventlet
 import eventlet.wsgi
 import getopt

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -7,7 +7,6 @@ from random import randint
 from loguru import logger
 
 from changedetectionio.content_fetchers.base import manage_user_agent
-from changedetectionio.safe_jinja import render as jinja_render
 
 # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
 # 0- off, 1- on
@@ -65,12 +64,14 @@ class steppable_browser_interface():
         action_handler = getattr(self, "action_" + call_action_name)
 
         # Support for Jinja2 variables in the value and selector
+        from jinja2 import Environment
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
 
         if selector and ('{%' in selector or '{{' in selector):
-            selector = jinja_render(template_str=selector)
+            selector = str(jinja2_env.from_string(selector).render())
 
         if optional_value and ('{%' in optional_value or '{{' in optional_value):
-            optional_value = jinja_render(template_str=optional_value)
+            optional_value = str(jinja2_env.from_string(optional_value).render())
 
         action_handler(selector, optional_value)
         self.page.wait_for_timeout(1.5 * 1000)

changedetectionio/blueprint/check_proxies/__init__.py

@@ -31,9 +31,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         import time
         from changedetectionio.content_fetchers import exceptions as content_fetcher_exceptions
         from changedetectionio.processors import text_json_diff
-        from changedetectionio.safe_jinja import render as jinja_render
 
         status = {'status': '', 'length': 0, 'text': ''}
+        from jinja2 import Environment, BaseLoader
 
         contents = ''
         now = time.time()
@@ -64,9 +64,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             status.update({'status': 'OK', 'length': len(contents), 'text': ''})
 
         if status.get('text'):
-            # parse 'text' as text for safety
+            status['text'] = Environment(loader=BaseLoader()).from_string('{{text|e}}').render({'text': status['text']})
-            v = {'text': status['text']}
-            status['text'] = jinja_render(template_str='{{text|e}}', **v)
 
         status['time'] = "{:.2f}s".format(time.time() - now)
 

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -1,7 +1,7 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
-{% from '_common_fields.html' import render_common_settings_form %}
+{% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
     const notification_base_url="{{url_for('ajax_callback_send_notification_test', mode="group-settings")}}";
 </script>

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_simple_field, render_field %}
+{% from '_helpers.jinja' import render_simple_field, render_field %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 
 <div class="box">

changedetectionio/content_fetchers/base.py

@@ -123,7 +123,8 @@ class Fetcher():
     def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
         from playwright._impl._errors import TimeoutError, Error
-        from changedetectionio.safe_jinja import render as jinja_render
+        from jinja2 import Environment
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
 
         step_n = 0
 
@@ -142,9 +143,9 @@ class Fetcher():
                     selector = step['selector']
                     # Support for jinja2 template in step values, with date module added
                     if '{%' in step['optional_value'] or '{{' in step['optional_value']:
-                        optional_value = jinja_render(template_str=step['optional_value'])
+                        optional_value = str(jinja2_env.from_string(step['optional_value']).render())
                     if '{%' in step['selector'] or '{{' in step['selector']:
-                        selector = jinja_render(template_str=step['selector'])
+                        selector = str(jinja2_env.from_string(step['selector']).render())
 
                     getattr(interface, "call_action")(action_name=step['operation'],
                                                       selector=selector,

changedetectionio/flask_app.py

@@ -5,11 +5,11 @@ import os
 import queue
 import threading
 import time
-from .safe_jinja import render as jinja_render
-from changedetectionio.strtobool import strtobool
 from copy import deepcopy
+from changedetectionio.strtobool import strtobool
 from functools import wraps
 from threading import Event
+
 import flask_login
 import pytz
 import timeago
@@ -319,6 +319,8 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/rss", methods=['GET'])
     def rss():
+        from jinja2 import Environment, BaseLoader
+        jinja2_env = Environment(loader=BaseLoader)
         now = time.time()
         # Always requires token set
         app_rss_token = datastore.data['settings']['application'].get('rss_access_token')
@@ -386,7 +388,7 @@ def changedetection_app(config=None, datastore_o=None):
                 # @todo Make this configurable and also consider html-colored markup
                 # @todo User could decide if <link> goes to the diff page, or to the watch link
                 rss_template = "<html><body>\n<h4><a href=\"{{watch_url}}\">{{watch_title}}</a></h4>\n<p>{{html_diff}}</p>\n</body></html>\n"
-                content = jinja_render(template_str=rss_template, watch_title=watch_title, html_diff=html_diff, watch_url=watch.link)
+                content = jinja2_env.from_string(rss_template).render(watch_title=watch_title, html_diff=html_diff, watch_url=watch.link)
 
                 fe.content(content=content, type='CDATA')
 
@@ -450,8 +452,6 @@ def changedetection_app(config=None, datastore_o=None):
             if search_q:
                 if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
                     sorted_watches.append(watch)
-                elif watch.get('last_error') and search_q in watch.get('last_error').lower():
-                    sorted_watches.append(watch)
             else:
                 sorted_watches.append(watch)
 
@@ -619,6 +619,7 @@ def changedetection_app(config=None, datastore_o=None):
         from .blueprint.browser_steps.browser_steps import browser_step_ui_config
         from . import processors
 
+        using_default_check_time = True
         # More for testing, possible to return the first/only
         if not datastore.data['watching'].keys():
             flash("No watches to edit", "error")
@@ -643,6 +644,10 @@ def changedetection_app(config=None, datastore_o=None):
         # be sure we update with a copy instead of accidently editing the live object by reference
         default = deepcopy(datastore.data['watching'][uuid])
 
+        # Show system wide default if nothing configured
+        if all(value == 0 or value == None for value in datastore.data['watching'][uuid]['time_between_check'].values()):
+            default['time_between_check'] = deepcopy(datastore.data['settings']['requests']['time_between_check'])
+
         # Defaults for proxy choice
         if datastore.proxy_list is not None:  # When enabled
             # @todo
@@ -680,8 +685,18 @@ def changedetection_app(config=None, datastore_o=None):
 
             if request.args.get('unpause_on_save'):
                 extra_update_obj['paused'] = False
+            # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
+            # Assume we use the default value, unless something relevant is different, then use the form value
+            # values could be None, 0 etc.
+            # Set to None unless the next for: says that something is different
+            extra_update_obj['time_between_check'] = dict.fromkeys(form.time_between_check.data)
+            for k, v in form.time_between_check.data.items():
+                if v and v != datastore.data['settings']['requests']['time_between_check'][k]:
+                    extra_update_obj['time_between_check'] = form.time_between_check.data
+                    using_default_check_time = False
+                    break
+
 
-            extra_update_obj['time_between_check'] = form.time_between_check.data
 
              # Ignore text
             form_ignore_text = form.ignore_text.data
@@ -762,6 +777,7 @@ def changedetection_app(config=None, datastore_o=None):
                                      extra_title=f" - Edit - {watch.label}",
                                      form=form,
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
+                                     has_empty_checktime=using_default_check_time,
                                      has_extra_headers_file=len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
                                      has_special_tag_options=_watch_has_tag_options_set(watch=watch),
                                      is_html_webdriver=is_html_webdriver,
@@ -847,13 +863,11 @@ def changedetection_app(config=None, datastore_o=None):
                 flash("An error occurred, please see below.", "error")
 
         output = render_template("settings.html",
-                                 api_key=datastore.data['settings']['application'].get('api_access_token'),
-                                 emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
                                  form=form,
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
-                                 min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)),
+                                 api_key=datastore.data['settings']['application'].get('api_access_token'),
-                                 settings_application=datastore.data['settings']['application']
+                                 emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
-                                 )
+                                 settings_application=datastore.data['settings']['application'])
 
         return output
 
@@ -1654,14 +1668,14 @@ def notification_runner():
             # Trim the log length
             notification_debug_log = notification_debug_log[-100:]
 
-# Threaded runner, look for new watches to feed into the Queue.
+# Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
     import random
     from changedetectionio import update_worker
 
     proxy_last_called_time = {}
 
-    recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3))
+    recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 20))
     logger.debug(f"System env MINIMUM_SECONDS_RECHECK_TIME {recheck_time_minimum_seconds}")
 
     # Spin up Workers that do the fetching
@@ -1715,7 +1729,9 @@ def ticker_thread_check_time_launch_checks():
                 continue
 
             # If they supplied an individual entry minutes to threshold.
-            threshold = recheck_time_system_seconds if watch.get('time_between_check_use_default') else watch.threshold_seconds()
+
+            watch_threshold_seconds = watch.threshold_seconds()
+            threshold = watch_threshold_seconds if watch_threshold_seconds > 0 else recheck_time_system_seconds
 
             # #580 - Jitter plus/minus amount of time to make the check seem more random to the server
             jitter = datastore.data['settings']['requests'].get('jitter_seconds', 0)

changedetectionio/forms.py

@@ -236,26 +236,21 @@ class ValidateJinja2Template(object):
     def __call__(self, form, field):
         from changedetectionio import notification
 
-        from jinja2 import BaseLoader, TemplateSyntaxError, UndefinedError
+        from jinja2 import Environment, BaseLoader, TemplateSyntaxError, UndefinedError
-        from jinja2.sandbox import ImmutableSandboxedEnvironment
         from jinja2.meta import find_undeclared_variables
-        import jinja2.exceptions
 
-        # Might be a list of text, or might be just text (like from the apprise url list)
-        joined_data = ' '.join(map(str, field.data)) if isinstance(field.data, list) else f"{field.data}"
 
         try:
-            jinja2_env = ImmutableSandboxedEnvironment(loader=BaseLoader)
+            jinja2_env = Environment(loader=BaseLoader)
             jinja2_env.globals.update(notification.valid_tokens)
-            jinja2_env.from_string(joined_data).render()
+
+            rendered = jinja2_env.from_string(field.data).render()
         except TemplateSyntaxError as e:
             raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
         except UndefinedError as e:
             raise ValidationError(f"A variable or function is not defined: {e}") from e
-        except jinja2.exceptions.SecurityError as e:
-            raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
 
-        ast = jinja2_env.parse(joined_data)
+        ast = jinja2_env.parse(field.data)
         undefined = ", ".join(find_undeclared_variables(ast))
         if undefined:
             raise ValidationError(
@@ -420,7 +415,7 @@ class quickWatchForm(Form):
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
 
-    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
+    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_format = SelectField('Notification format', choices=valid_notification_formats.keys())
@@ -453,7 +448,6 @@ class watchForm(commonSettingsForm):
     tags = StringTagUUID('Group tag', [validators.Optional()], default='')
 
     time_between_check = FormField(TimeBetweenCheckForm)
-    time_between_check_use_default = BooleanField('Use global settings for time between check', default=False)
 
     include_filters = StringListField('CSS/JSONPath/JQ/XPath Filters', [ValidateCSSJSONXPATHInput()], default='')
 
@@ -505,9 +499,11 @@ class watchForm(commonSettingsForm):
             result = False
 
         # Attempt to validate jinja2 templates in the URL
+        from jinja2 import Environment
+        # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
         try:
-            from changedetectionio.safe_jinja import render as jinja_render
+            ready_url = str(jinja2_env.from_string(self.url.data).render())
-            jinja_render(template_str=self.url.data)
         except Exception as e:
             self.url.errors.append('Invalid template syntax')
             result = False

changedetectionio/model/Watch.py

@@ -1,6 +1,4 @@
 from changedetectionio.strtobool import strtobool
-from changedetectionio.safe_jinja import render as jinja_render
-
 import os
 import re
 import time
@@ -12,7 +10,7 @@ from loguru import logger
 # file:// is further checked by ALLOW_FILE_URI
 SAFE_PROTOCOL_REGEX='^(http|https|ftp|file):'
 
-minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3))
+minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
 mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
 
 from changedetectionio.notification import (
@@ -69,7 +67,6 @@ base_config = {
     # Requires setting to None on submit if it's the same as the default
     # Should be all None by default, so we use the system default in this case.
     'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
-    'time_between_check_use_default': True,
     'title': None,
     'trigger_text': [],  # List of text or regex to wait for until a change is detected
     'url': '',
@@ -140,11 +137,12 @@ class model(dict):
 
         ready_url = url
         if '{%' in url or '{{' in url:
+            from jinja2 import Environment
             # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
+            jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
             try:
-                ready_url = jinja_render(template_str=url)
+                ready_url = str(jinja2_env.from_string(url).render())
             except Exception as e:
-                logger.critical(f"Invalid URL template for: '{url}' - {str(e)}")
                 from flask import (
                     flash, Markup, url_for
                 )

changedetectionio/notification.py

@@ -1,5 +1,6 @@
 import apprise
 import time
+from jinja2 import Environment, BaseLoader
 from apprise import NotifyFormat
 import json
 from loguru import logger
@@ -115,7 +116,6 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
 
 def process_notification(n_object, datastore):
 
-    from .safe_jinja import render as jinja_render
     now = time.time()
     if n_object.get('notification_timestamp'):
         logger.trace(f"Time since queued {now-n_object['notification_timestamp']:.3f}s")
@@ -123,9 +123,9 @@ def process_notification(n_object, datastore):
     notification_parameters = create_notification_parameters(n_object, datastore)
 
     # Get the notification body from datastore
-    n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
+    jinja2_env = Environment(loader=BaseLoader)
-    n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
+    n_body = jinja2_env.from_string(n_object.get('notification_body', '')).render(**notification_parameters)
-
+    n_title = jinja2_env.from_string(n_object.get('notification_title', '')).render(**notification_parameters)
     n_format = valid_notification_formats.get(
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
@@ -157,7 +157,7 @@ def process_notification(n_object, datastore):
                 continue
 
             logger.info(">> Process Notification: AppRise notifying {}".format(url))
-            url = jinja_render(template_str=url, **notification_parameters)
+            url = jinja2_env.from_string(url).render(**notification_parameters)
 
             # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
             # Because different notifications may require different pre-processing, run each sequentially :(

changedetectionio/safe_jinja.py

@@ -1,18 +0,0 @@
-"""
-Safe Jinja2 render with max payload sizes
-
-See https://jinja.palletsprojects.com/en/3.1.x/sandbox/#security-considerations
-"""
-
-import jinja2.sandbox
-import typing as t
-import os
-
-JINJA2_MAX_RETURN_PAYLOAD_SIZE = 1024 * int(os.getenv("JINJA2_MAX_RETURN_PAYLOAD_SIZE_KB", 1024 * 10))
-
-
-def render(template_str, **args: t.Any) -> str:
-    jinja2_env = jinja2.sandbox.ImmutableSandboxedEnvironment(extensions=['jinja2_time.TimeExtension'])
-    output = jinja2_env.from_string(template_str).render(args)
-    return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]
-

changedetectionio/static/js/watch-settings.js

@@ -1,17 +1,3 @@
-function toggleOpacity(checkboxSelector, fieldSelector) {
-    const checkbox = document.querySelector(checkboxSelector);
-    const fields = document.querySelectorAll(fieldSelector);
-    function updateOpacity() {
-        const opacityValue = checkbox.checked ? 0.6 : 1;
-        fields.forEach(field => {
-            field.style.opacity = opacityValue;
-        });
-    }
-    // Initial setup
-    updateOpacity();
-    checkbox.addEventListener('change', updateOpacity);
-}
-
 $(document).ready(function () {
     $('#notification-setting-reset-to-default').click(function (e) {
         $('#notification_title').val('');
@@ -24,7 +10,4 @@ $(document).ready(function () {
         e.preventDefault();
         $('#notification-tokens-info').toggle();
     });
-
-    toggleOpacity('#time_between_check_use_default', '#time_between_check');
 });
-

changedetectionio/static/styles/diff.css

@@ -68,7 +68,7 @@
   --color-last-checked: #bbb;
   --color-text-footer: #444;
   --color-border-watch-table-cell: #eee;
-  --color-text-watch-tag-list: rgba(231, 0, 105, 0.4);
+  --color-text-watch-tag-list: #e70069;
   --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
   --color-background-new-watch-input: var(--color-white);
   --color-text-new-watch-input: var(--color-text);
@@ -111,7 +111,7 @@ html[data-darkmode="true"] {
   --color-background-input: var(--color-grey-350);
   --color-text-input-description: var(--color-grey-600);
   --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: rgba(250, 62, 146, 0.4);
+  --color-text-watch-tag-list: #fa3e92;
   --color-background-code: var(--color-grey-200);
   --color-background-tab: rgba(0, 0, 0, 0.2);
   --color-background-tab-hover: rgba(0, 0, 0, 0.5);

changedetectionio/static/styles/scss/parts/_variables.scss

@@ -75,7 +75,7 @@
   --color-text-footer: #444;
   --color-border-watch-table-cell: #eee;
 
-  --color-text-watch-tag-list: rgba(231, 0, 105, 0.4);
+  --color-text-watch-tag-list: #e70069;
   --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
   --color-background-new-watch-input: var(--color-white);
   --color-text-new-watch-input: var(--color-text);
@@ -127,7 +127,7 @@ html[data-darkmode="true"] {
   --color-background-input: var(--color-grey-350);
   --color-text-input-description: var(--color-grey-600);
   --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: rgba(250, 62, 146, 0.4);
+  --color-text-watch-tag-list: #fa3e92;
   --color-background-code: var(--color-grey-200);
 
   --color-background-tab: rgba(0, 0, 0, 0.2);

changedetectionio/static/styles/scss/styles.scss

@@ -187,11 +187,8 @@ code {
 }
 
 .watch-tag-list {
-  color: var(--color-white);
+  color: var(--color-text-watch-tag-list);
   white-space: nowrap;
-  background: var(--color-text-watch-tag-list);
-  border-radius: 5px;
-  padding: 2px 5px;
 }
 
 .box {
@@ -928,26 +925,23 @@ body.full-width {
       font-size: .875em;
     }
   }
-}
+  .text-filtering {
-
+    h3 {
-.border-fieldset {
+      margin-top: 0;
-  h3 {
+    }
-    margin-top: 0;
+    border: 1px solid #ccc;
-  }
+    padding: 1rem;
-  border: 1px solid #ccc;
+    border-radius: 5px;
-  padding: 1rem;
+    margin-bottom: 1rem;
-  border-radius: 5px;
+    fieldset:last-of-type {
-  margin-bottom: 1rem;
-  fieldset:last-of-type {
-    padding-bottom: 0;
-    .pure-control-group {
       padding-bottom: 0;
+      .pure-control-group {
+        padding-bottom: 0;
+      }
     }
   }
 }
 
-
-
 ul {
   padding-left: 1em;
   padding-top: 0px;

changedetectionio/static/styles/styles.css

@@ -284,7 +284,7 @@ ul#requests-extra_browsers {
   --color-last-checked: #bbb;
   --color-text-footer: #444;
   --color-border-watch-table-cell: #eee;
-  --color-text-watch-tag-list: rgba(231, 0, 105, 0.4);
+  --color-text-watch-tag-list: #e70069;
   --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
   --color-background-new-watch-input: var(--color-white);
   --color-text-new-watch-input: var(--color-text);
@@ -327,7 +327,7 @@ html[data-darkmode="true"] {
   --color-background-input: var(--color-grey-350);
   --color-text-input-description: var(--color-grey-600);
   --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: rgba(250, 62, 146, 0.4);
+  --color-text-watch-tag-list: #fa3e92;
   --color-background-code: var(--color-grey-200);
   --color-background-tab: rgba(0, 0, 0, 0.2);
   --color-background-tab-hover: rgba(0, 0, 0, 0.5);
@@ -532,11 +532,8 @@ code {
     margin: 0 3px 0 5px; }
 
 .watch-tag-list {
-  color: var(--color-white);
+  color: var(--color-text-watch-tag-list);
-  white-space: nowrap;
+  white-space: nowrap; }
-  background: var(--color-text-watch-tag-list);
-  border-radius: 5px;
-  padding: 2px 5px; }
 
 .box {
   max-width: 80%;
@@ -1041,18 +1038,17 @@ body.full-width .edit-form {
     color: var(--color-text-input-description); }
     .edit-form .pure-form-message-inline code {
       font-size: .875em; }
-
+  .edit-form .text-filtering {
-.border-fieldset {
+    border: 1px solid #ccc;
-  border: 1px solid #ccc;
+    padding: 1rem;
-  padding: 1rem;
+    border-radius: 5px;
-  border-radius: 5px;
+    margin-bottom: 1rem; }
-  margin-bottom: 1rem; }
+    .edit-form .text-filtering h3 {
-  .border-fieldset h3 {
+      margin-top: 0; }
-    margin-top: 0; }
+    .edit-form .text-filtering fieldset:last-of-type {
-  .border-fieldset fieldset:last-of-type {
-    padding-bottom: 0; }
-    .border-fieldset fieldset:last-of-type .pure-control-group {
       padding-bottom: 0; }
+      .edit-form .text-filtering fieldset:last-of-type .pure-control-group {
+        padding-bottom: 0; }
 
 ul {
   padding-left: 1em;

changedetectionio/store.py

@@ -872,16 +872,3 @@ class ChangeDetectionStore:
                         self.__data["watching"][awatch]['include_filters'][num] = 'xpath1:' + selector
                     if selector.startswith('xpath:'):
                         self.__data["watching"][awatch]['include_filters'][num] = selector.replace('xpath:', 'xpath1:', 1)
-
-    # Use more obvious default time setting
-    def update_15(self):
-        for uuid in self.__data["watching"]:
-            if self.__data["watching"][uuid]['time_between_check'] == self.__data['settings']['requests']['time_between_check']:
-                # What the old logic was, which was pretty confusing
-                self.__data["watching"][uuid]['time_between_check_use_default'] = True
-            elif all(value is None or value == 0 for value in self.__data["watching"][uuid]['time_between_check'].values()):
-                self.__data["watching"][uuid]['time_between_check_use_default'] = True
-            else:
-                # Something custom here
-                self.__data["watching"][uuid]['time_between_check_use_default'] = False
-

changedetectionio/templates/IMPORTANT.md

@@ -1,6 +0,0 @@
-# Important notes about templates
-
-Template names should always end in ".html", ".htm", ".xml", ".xhtml", ".svg", even the `import`'ed templates.
-
-Jinja2's `def select_jinja_autoescape(self, filename: str) -> bool:` will check the filename extension and enable autoescaping
-

changedetectionio/templates/_common_fields.jinja

@@ -1,5 +1,5 @@
 
-{% from '_helpers.html' import render_field %}
+{% from '_helpers.jinja' import render_field %}
 
 {% macro render_common_settings_form(form, emailprefix, settings_application) %}
                         <div class="pure-control-group">

changedetectionio/templates/diff.html

@@ -1,5 +1,5 @@
 {% extends 'base.html' %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% block content %}
 <script>
     const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";

changedetectionio/templates/edit.html

@@ -1,7 +1,7 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
-{% from '_common_fields.html' import render_common_settings_form %}
+{% from '_common_fields.jinja' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script>
@@ -87,9 +87,15 @@
                         {{ render_field(form.tags) }}
                         <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
                     </div>
-                    <div class="pure-control-group time-between-check border-fieldset">
+                    <div class="pure-control-group">
                         {{ render_field(form.time_between_check, class="time-check-widget") }}
-                        {{ render_checkbox_field(form.time_between_check_use_default, class="use-default-timecheck") }}
+                        {% if has_empty_checktime %}
+                        <span class="pure-form-message-inline">Currently using the <a
+                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
+                        {% else %}
+                        <span class="pure-form-message-inline">Set to blank to use the <a
+                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>.</span>
+                        {% endif %}
                     </div>
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.extract_title_as_title) }}
@@ -324,7 +330,7 @@ nav
                         </ul>
                       </span>
                 </fieldset>
-                <div class="text-filtering border-fieldset">
+                <div class="text-filtering">
                 <fieldset class="pure-group" id="text-filtering-type-options">
                     <h3>Text filtering</h3>
                         Limit trigger/ignore/block/extract to;<br>
@@ -481,17 +487,13 @@ Unavailable") }}
                             <td>{{ "{:,}".format(watch.history|length) }}</td>
                         </tr>
                         <tr>
-                            <td>Last fetch duration</td>
+                            <td>Last fetch time</td>
                             <td>{{ watch.fetch_time }}s</td>
                         </tr>
                         <tr>
                             <td>Notification alert count</td>
                             <td>{{ watch.notification_alert_count }}</td>
                         </tr>
-                        <tr>
-                            <td>Server type reply</td>
-                            <td>{{ watch.get('remote_server_reply') }}</td>
-                        </tr>
                         </tbody>
                     </table>
                 </div>

changedetectionio/templates/import.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field %}
+{% from '_helpers.jinja' import render_field %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <div class="edit-form monospaced-textarea">
 

changedetectionio/templates/settings.html

@@ -1,8 +1,8 @@
 {% extends 'base.html' %}
 
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
-{% from '_common_fields.html' import render_common_settings_form %}
+{% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
     const notification_base_url="{{url_for('ajax_callback_send_notification_test', mode="global-settings")}}";
 {% if emailprefix %}
@@ -31,7 +31,7 @@
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }}
-                        <span class="pure-form-message-inline">Default recheck time for all watches, current system minimum is <i>{{min_system_recheck_seconds}}</i> seconds (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Misc-system-settings#enviroment-variables">more info</a>).</span>
+                        <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}

changedetectionio/templates/watch-overview.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_simple_field, render_field, render_nolabel_field, sort_by_title %}
+{% from '_helpers.jinja' import render_simple_field, render_field, render_nolabel_field, sort_by_title %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 
@@ -169,7 +169,7 @@
                 <td>
                     <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                        class="recheck pure-button pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
-                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
+                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button pure-button-primary">Edit</a>
                     {% if watch.history_n >= 2 %}
 
                         {%  if is_unviewed %}

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python3
-import os.path
+
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
@@ -107,6 +107,7 @@ def test_check_add_line_contains_trigger(client, live_server):
     #live_server_setup(live_server)
 
     # Give the endpoint time to spin up
+    time.sleep(1)
     test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://') + "?xxx={{ watch_url }}"
 
     res = client.post(
@@ -165,7 +166,6 @@ def test_check_add_line_contains_trigger(client, live_server):
 
     # Takes a moment for apprise to fire
     time.sleep(3)
-    assert os.path.isfile("test-datastore/notification.txt"), "Notification fired because I can see the output file"
     with open("test-datastore/notification.txt", 'r') as f:
         response= f.read()
         assert '-Oh yes please-' in response

changedetectionio/tests/test_jinja2.py

@@ -2,15 +2,15 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from .util import live_server_setup
 
 
-def test_setup(client, live_server):
-    live_server_setup(live_server)
-
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
 def test_jinja2_in_url_query(client, live_server):
-    #live_server_setup(live_server)
+    live_server_setup(live_server)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_return_query', _external=True)
@@ -24,35 +24,10 @@ def test_jinja2_in_url_query(client, live_server):
         follow_redirects=True
     )
     assert b"Watch added" in res.data
-    wait_for_all_checks(client)
+    time.sleep(3)
-
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True
     )
     assert b'date=2' in res.data
-
-# https://techtonics.medium.com/secure-templating-with-jinja2-understanding-ssti-and-jinja2-sandbox-environment-b956edd60456
-def test_jinja2_security_url_query(client, live_server):
-    #live_server_setup(live_server)
-
-    # Add our URL to the import page
-    test_url = url_for('test_return_query', _external=True)
-
-    # because url_for() will URL-encode the var, but we dont here
-    full_url = "{}?{}".format(test_url,
-                              "date={{ ''.__class__.__mro__[1].__subclasses__()}}", )
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": full_url, "tags": "test"},
-        follow_redirects=True
-    )
-    assert b"Watch added" in res.data
-    wait_for_all_checks(client)
-
-    # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
-    assert b'is invalid and cannot be used' in res.data
-    # Some of the spewed output from the subclasses
-    assert b'dict_values' not in res.data

changedetectionio/tests/test_security.py

@@ -2,11 +2,9 @@ from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 import time
 
-def test_setup(client, live_server):
-    live_server_setup(live_server)
 
 def test_bad_access(client, live_server):
-    #live_server_setup(live_server)
+    live_server_setup(live_server)
     res = client.post(
         url_for("import_page"),
         data={"urls": 'https://localhost'},
@@ -65,25 +63,4 @@ def test_bad_access(client, live_server):
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
 
-    assert b'file:// type access is denied for security reasons.' in res.data
+    assert b'file:// type access is denied for security reasons.' in res.data
-
-def test_xss(client, live_server):
-    #live_server_setup(live_server)
-    from changedetectionio.notification import (
-        default_notification_format
-    )
-    # the template helpers were named .jinja which meant they were not having jinja2 autoescape enabled.
-    res = client.post(
-        url_for("settings_page"),
-        data={"application-notification_urls": '"><img src=x onerror=alert(document.domain)>',
-              "application-notification_title": '"><img src=x onerror=alert(document.domain)>',
-              "application-notification_body": '"><img src=x onerror=alert(document.domain)>',
-              "application-notification_format": default_notification_format,
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"<img src=x onerror=alert(" not in res.data
-    assert b"&lt;img" in res.data
-

changedetectionio/tests/test_watch_fields_storage.py

@@ -54,3 +54,102 @@ def test_check_watch_field_storage(client, live_server):
     assert b"woohoo" in res.data
     assert b"curl: foo" in res.data
 
+
+
+# Re https://github.com/dgtlmoon/changedetection.io/issues/110
+def test_check_recheck_global_setting(client, live_server):
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "requests-time_between_check-minutes": 1566,
+               'application-fetch_backend': "html_requests"
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Now add a record
+
+    test_url = "http://somerandomsitewewatch.com"
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Now visit the edit page, it should have the default minutes
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should show the default minutes
+    assert b"change to another value if you want to be specific" in res.data
+    assert b"1566" in res.data
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "requests-time_between_check-minutes": 222,
+                'application-fetch_backend': "html_requests"
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should show the default minutes
+    assert b"change to another value if you want to be specific" in res.data
+    assert b"222" in res.data
+
+    # Now change it specifically, it should show the new minutes
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"url": test_url,
+              "time_between_check-minutes": 55,
+              'fetch_backend': "html_requests"
+              },
+        follow_redirects=True
+    )
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"55" in res.data
+
+    # Now submit an empty field, it should give back the default global minutes
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "requests-time_between_check-minutes": 666,
+                "application-fetch_backend": "html_requests"
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"url": test_url,
+              "time_between_check-minutes": "",
+              'fetch_backend': "html_requests"
+              },
+        follow_redirects=True
+    )
+
+    assert b"Updated watch." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"666" in res.data

changedetectionio/tests/unit/test_jinja2_security.py

@@ -1,57 +0,0 @@
-#!/usr/bin/python3
-
-# run from dir above changedetectionio/ dir
-# python3 -m unittest changedetectionio.tests.unit.test_jinja2_security
-
-import unittest
-from changedetectionio import safe_jinja
-
-
-# mostly
-class TestJinja2SSTI(unittest.TestCase):
-
-    def test_exception(self):
-        import jinja2
-
-        # Where sandbox should kick in
-        attempt_list = [
-            "My name is {{ self.__init__.__globals__.__builtins__.__import__('os').system('id') }}",
-            "{{ self._TemplateReference__context.cycler.__init__.__globals__.os }}",
-            "{{ self.__init__.__globals__.__builtins__.__import__('os').popen('id').read() }}",
-            "{{cycler.__init__.__globals__.os.popen('id').read()}}",
-            "{{joiner.__init__.__globals__.os.popen('id').read()}}",
-            "{{namespace.__init__.__globals__.os.popen('id').read()}}",
-            "{{ ''.__class__.__mro__[2].__subclasses__()[40]('/tmp/hello.txt', 'w').write('Hello here !') }}",
-            "My name is {{ self.__init__.__globals__ }}",
-            "{{ dict.__base__.__subclasses__() }}"
-        ]
-        for attempt in attempt_list:
-            with self.assertRaises(jinja2.exceptions.SecurityError):
-                safe_jinja.render(attempt)
-
-    def test_exception_debug_calls(self):
-        import jinja2
-        # Where sandbox should kick in - configs and debug calls
-        attempt_list = [
-            "{% debug %}",
-        ]
-        for attempt in attempt_list:
-            # Usually should be something like 'Encountered unknown tag 'debug'.'
-            with self.assertRaises(jinja2.exceptions.TemplateSyntaxError):
-                safe_jinja.render(attempt)
-
-    # https://book.hacktricks.xyz/pentesting-web/ssti-server-side-template-injection/jinja2-ssti#accessing-global-objects
-    def test_exception_empty_calls(self):
-        import jinja2
-        attempt_list = [
-            "{{config}}",
-            "{{ debug }}"
-            "{{[].__class__}}",
-        ]
-        for attempt in attempt_list:
-            self.assertEqual(len(safe_jinja.render(attempt)), 0, f"string test '{attempt}' is correctly empty")
-
-
-
-if __name__ == '__main__':
-    unittest.main()

changedetectionio/tests/util.py

@@ -116,7 +116,7 @@ def extract_UUID_from_client(client):
     )
     # <span id="api-key">{{api_key}}</span>
 
-    m = re.search('edit/(.+?)[#"]', str(res.data))
+    m = re.search('edit/(.+?)"', str(res.data))
     uuid = m.group(1)
     return uuid.strip()
 

docker-compose.yml

@@ -54,9 +54,7 @@ services:
   #        
   #        Default number of parallel/concurrent fetchers
   #      - FETCH_WORKERS=10
-  #
+
-  #        Absolute minimum seconds to recheck, overrides any watch minimum, change to 0 to disable
-  #      - MINIMUM_SECONDS_RECHECK_TIME=3
       # Comment out ports: when using behind a reverse proxy , enable networks: etc.
       ports:
         - 5000:5000

requirements.txt

@@ -41,7 +41,7 @@ apprise~=1.7.4
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 # and 2.0.0 https://github.com/dgtlmoon/changedetection.io/issues/2241 not yet compatible
 # use v1.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
-paho-mqtt>=1.6.1,<2.0.0
+paho-mqtt < 2.0.0
 
 # This mainly affects some ARM builds, which unlike the other builds ignores "ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1"
 # so without this pinning, the newer versions on ARM will forcefully try to build rust, which results in "rust compiler not found"