0.45.21

* CVE-2024-32651 - Security fix - Server Side Template Injection in Jinja2 allows Remote Command Execution

* use ImmutableSandboxedEnvironment also in validation

.github/dependabot.yml

@@ -4,6 +4,10 @@ updates:
     directory: /
     schedule:
       interval: "weekly"
+    "caronc/apprise":
+      versioning-strategy: "increase"
+      schedule:
+        interval: "daily"
     groups:
       all:
         patterns:

.github/workflows/test-only.yml

@@ -59,6 +59,7 @@ jobs:
           echo "run test with unittest"
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
           
           # All tests
           echo "run test with pytest"

CONTRIBUTING.md

@@ -2,7 +2,7 @@ Contributing is always welcome!
 
 I am no professional flask developer, if you know a better way that something can be done, please let me know!
 
-Otherwise, it's always best to PR into the `dev` branch.
+Otherwise, it's always best to PR into the `master` branch.
 
 Please be sure that all new functionality has a matching test!
 

MANIFEST.in

@@ -1,8 +1,8 @@
 recursive-include changedetectionio/api *
 recursive-include changedetectionio/blueprint *
+recursive-include changedetectionio/content_fetchers *
 recursive-include changedetectionio/model *
 recursive-include changedetectionio/processors *
-recursive-include changedetectionio/res *
 recursive-include changedetectionio/static *
 recursive-include changedetectionio/templates *
 recursive-include changedetectionio/tests *

README.md

@@ -91,6 +91,14 @@ We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) glob
 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
+### We have a Chrome extension!
+
+Easily add the current web page to your changedetection.io tool, simply install the extension and click "Sync" to connect it to your existing changedetection.io install.
+
+[<img src="./docs/chrome-extension-screenshot.png" style="max-width:80%;" alt="Chrome Extension to easily add the current web-page to detect a change."  title="Chrome Extension to easily add the current web-page to detect a change."  />](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
+
+[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
+
 ## Installation
 
 ### Docker

changedetectionio/__init__.py

@@ -2,12 +2,12 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.45.14'
+__version__ = '0.45.21'
 
-from distutils.util import strtobool
+from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
 import os
-#os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
+os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
 import eventlet
 import eventlet.wsgi
 import getopt

changedetectionio/api/api_v1.py

@@ -1,5 +1,5 @@
 import os
-from distutils.util import strtobool
+from changedetectionio.strtobool import strtobool
 
 from flask_expects_json import expects_json
 from changedetectionio import queuedWatchMetaData

changedetectionio/blueprint/browser_steps/__init__.py

@@ -12,7 +12,7 @@
 #
 #
 
-from distutils.util import strtobool
+from changedetectionio.strtobool import strtobool
 from flask import Blueprint, request, make_response
 import os
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -7,6 +7,7 @@ from random import randint
 from loguru import logger
 
 from changedetectionio.content_fetchers.base import manage_user_agent
+from changedetectionio.safe_jinja import render as jinja_render
 
 # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
 # 0- off, 1- on
@@ -64,14 +65,12 @@ class steppable_browser_interface():
         action_handler = getattr(self, "action_" + call_action_name)
 
         # Support for Jinja2 variables in the value and selector
-        from jinja2 import Environment
-        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
 
         if selector and ('{%' in selector or '{{' in selector):
-            selector = str(jinja2_env.from_string(selector).render())
+            selector = jinja_render(template_str=selector)
 
         if optional_value and ('{%' in optional_value or '{{' in optional_value):
-            optional_value = str(jinja2_env.from_string(optional_value).render())
+            optional_value = jinja_render(template_str=optional_value)
 
         action_handler(selector, optional_value)
         self.page.wait_for_timeout(1.5 * 1000)

changedetectionio/blueprint/check_proxies/__init__.py

@@ -31,9 +31,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         import time
         from changedetectionio.content_fetchers import exceptions as content_fetcher_exceptions
         from changedetectionio.processors import text_json_diff
+        from changedetectionio.safe_jinja import render as jinja_render
 
         status = {'status': '', 'length': 0, 'text': ''}
-        from jinja2 import Environment, BaseLoader
 
         contents = ''
         now = time.time()
@@ -64,7 +64,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             status.update({'status': 'OK', 'length': len(contents), 'text': ''})
 
         if status.get('text'):
-            status['text'] = Environment(loader=BaseLoader()).from_string('{{text|e}}').render({'text': status['text']})
+            # parse 'text' as text for safety
+            v = {'text': status['text']}
+            status['text'] = jinja_render(template_str='{{text|e}}', **v)
 
         status['time'] = "{:.2f}s".format(time.time() - now)
 

changedetectionio/blueprint/price_data_follower/__init__.py

@@ -1,5 +1,5 @@
 
-from distutils.util import strtobool
+from changedetectionio.strtobool import strtobool
 from flask import Blueprint, flash, redirect, url_for
 from flask_login import login_required
 from changedetectionio.store import ChangeDetectionStore

changedetectionio/blueprint/tags/__init__.py

@@ -12,9 +12,15 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         from .form import SingleTag
         add_form = SingleTag(request.form)
         sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
+
+        from collections import Counter
+
+        tag_count = Counter(tag for watch in datastore.data['watching'].values() if watch.get('tags') for tag in watch['tags'])
+
         output = render_template("groups-overview.html",
-                                 form=add_form,
                                  available_tags=sorted_tags,
+                                 form=add_form,
+                                 tag_count=tag_count
                                  )
 
         return output

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -3,7 +3,7 @@
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test', mode="group-settings")}}";
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -27,6 +27,7 @@
             <thead>
             <tr>
                 <th></th>
+                <th># Watches</th>
                 <th>Tag / Label name</th>
                 <th></th>
             </tr>
@@ -45,7 +46,8 @@
                 <td class="watch-controls">
                     <a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
                 </td>
-                <td class="title-col inline">{{tag.title}}</td>
+                <td>{{ "{:,}".format(tag_count[uuid]) if uuid in tag_count else 0 }}</td>
+                <td class="title-col inline"> <a href="{{url_for('index', tag=uuid) }}">{{ tag.title }}</a></td>
                 <td>
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">Edit</a>&nbsp;
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.delete', uuid=uuid) }}" title="Deletes and removes tag">Delete</a>

changedetectionio/content_fetchers/__init__.py

@@ -1,5 +1,5 @@
 import sys
-from distutils.util import strtobool
+from changedetectionio.strtobool import strtobool
 from loguru import logger
 from changedetectionio.content_fetchers.exceptions import BrowserStepsStepException
 import os

changedetectionio/content_fetchers/base.py

@@ -123,8 +123,7 @@ class Fetcher():
     def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
         from playwright._impl._errors import TimeoutError, Error
-        from jinja2 import Environment
-        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
+        from changedetectionio.safe_jinja import render as jinja_render
 
         step_n = 0
 
@@ -143,9 +142,9 @@ class Fetcher():
                     selector = step['selector']
                     # Support for jinja2 template in step values, with date module added
                     if '{%' in step['optional_value'] or '{{' in step['optional_value']:
-                        optional_value = str(jinja2_env.from_string(step['optional_value']).render())
+                        optional_value = jinja_render(template_str=step['optional_value'])
                     if '{%' in step['selector'] or '{{' in step['selector']:
-                        selector = str(jinja2_env.from_string(step['selector']).render())
+                        selector = jinja_render(template_str=step['selector'])
 
                     getattr(interface, "call_action")(action_name=step['operation'],
                                                       selector=selector,

changedetectionio/content_fetchers/res/puppeteer_fetch.js

@@ -18,7 +18,6 @@ module.exports = async ({page, context}) => {
 
     await page.setBypassCSP(true)
     await page.setExtraHTTPHeaders(req_headers);
-    var total_size = 0;
 
     if (user_agent) {
         await page.setUserAgent(user_agent);
@@ -43,88 +42,101 @@ module.exports = async ({page, context}) => {
         height: 768,
         deviceScaleFactor: 1,
     });
+
     await page.setRequestInterception(true);
-    await page.setCacheEnabled(false);
-
-
-    await page.evaluateOnNewDocument('navigator.serviceWorker.register = () => { console.warn("Service Worker registration blocked by Playwright")}');
-
-    await page.evaluateOnNewDocument(`
-   
-  const toBlob = HTMLCanvasElement.prototype.toBlob;
-  const toDataURL = HTMLCanvasElement.prototype.toDataURL;
-
-    HTMLCanvasElement.prototype.manipulate = function() {
-    console.warn("ma");
-    const {width, height} = this;
-    const context = this.getContext('2d');
-    var dt = new Date();
-    
-    const shift = {
-      'r': dt.getDay()-3,
-      'g': dt.getDay()-3,
-      'b': dt.getDay()-3
-    };
-    console.log(shift);
-    const matt = context.getImageData(0, 0, width, height);
-    for (let i = 0; i < height; i += Math.max(1, parseInt(height / 10))) {
-      for (let j = 0; j < width; j += Math.max(1, parseInt(width / 10))) {
-        const n = ((i * (width * 4)) + (j * 4));
-        matt.data[n + 0] = matt.data[n + 0] + shift.r;
-        matt.data[n + 1] = matt.data[n + 1] + shift.g;
-        matt.data[n + 2] = matt.data[n + 2] + shift.b;
-      }
+    if (disk_cache_dir) {
+        console.log(">>>>>>>>>>>>>>> LOCAL DISK CACHE ENABLED <<<<<<<<<<<<<<<<<<<<<");
     }
-    context.putImageData(matt, 0, 0);
-  };
+    const fs = require('fs');
+    const crypto = require('crypto');
 
-  Object.defineProperty(HTMLCanvasElement.prototype, 'toBlob', {
-    value: function() {
-    console.warn("toblob");
-      if (true) {
-        try {
-          this.manipulate();
+    function file_is_expired(file_path) {
+        if (!fs.existsSync(file_path)) {
+            return true;
         }
-        catch(e) {
-          console.warn('manipulation failed', e);
+        var stats = fs.statSync(file_path);
+        const now_date = new Date();
+        const expire_seconds = 300;
+        if ((now_date / 1000) - (stats.mtime.getTime() / 1000) > expire_seconds) {
+            console.log("CACHE EXPIRED: " + file_path);
+            return true;
         }
-      }
-      return toBlob.apply(this, arguments);
+        return false;
+
     }
-  });
-  Object.defineProperty(HTMLCanvasElement.prototype, 'toDataURL', {
-    value: function() {
-        console.warn("todata");
-      if (true) {
-        try {
-          this.manipulate();
+
+    page.on('request', async (request) => {
+        // General blocking of requests that waste traffic
+        if (block_url_list.some(substring => request.url().toLowerCase().includes(substring))) return request.abort();
+
+        if (disk_cache_dir) {
+            const url = request.url();
+            const key = crypto.createHash('md5').update(url).digest("hex");
+            const dir_path = disk_cache_dir + key.slice(0, 1) + '/' + key.slice(1, 2) + '/' + key.slice(2, 3) + '/';
+
+            // https://stackoverflow.com/questions/4482686/check-synchronously-if-file-directory-exists-in-node-js
+
+            if (fs.existsSync(dir_path + key)) {
+                console.log("* CACHE HIT , using - " + dir_path + key + " - " + url);
+                const cached_data = fs.readFileSync(dir_path + key);
+                // @todo headers can come from dir_path+key+".meta" json file
+                request.respond({
+                    status: 200,
+                    //contentType: 'text/html', //@todo
+                    body: cached_data
+                });
+                return;
+            }
         }
-        catch(e) {
-          console.warn('manipulation failed', e);
-        }
-      }
-      return toDataURL.apply(this, arguments);
-    }
-  });
-
-
-  Object.defineProperty(navigator, 'webdriver', {get: () => false});
-`)
-
-    await page.emulateTimezone('America/Chicago');
-
-    var r = await page.goto(url, {
-        waitUntil: 'load', timeout: 0
+        request.continue();
     });
 
-// https://github.com/puppeteer/puppeteer/issues/2479#issuecomment-408263504
-    if (r === null) {
-        r = await page.waitForResponse(() => true);
+
+    if (disk_cache_dir) {
+        page.on('response', async (response) => {
+            const url = response.url();
+            // Basic filtering for sane responses
+            if (response.request().method() != 'GET' || response.request().resourceType() == 'xhr' || response.request().resourceType() == 'document' || response.status() != 200) {
+                console.log("Skipping (not useful) - Status:" + response.status() + " Method:" + response.request().method() + " ResourceType:" + response.request().resourceType() + " " + url);
+                return;
+            }
+            if (no_cache_list.some(substring => url.toLowerCase().includes(substring))) {
+                console.log("Skipping (no_cache_list) - " + url);
+                return;
+            }
+            if (url.toLowerCase().includes('data:')) {
+                console.log("Skipping (embedded-data) - " + url);
+                return;
+            }
+            response.buffer().then(buffer => {
+                if (buffer.length > 100) {
+                    console.log("Cache - Saving " + response.request().method() + " - " + url + " - " + response.request().resourceType());
+
+                    const key = crypto.createHash('md5').update(url).digest("hex");
+                    const dir_path = disk_cache_dir + key.slice(0, 1) + '/' + key.slice(1, 2) + '/' + key.slice(2, 3) + '/';
+
+                    if (!fs.existsSync(dir_path)) {
+                        fs.mkdirSync(dir_path, {recursive: true})
+                    }
+
+                    if (fs.existsSync(dir_path + key)) {
+                        if (file_is_expired(dir_path + key)) {
+                            fs.writeFileSync(dir_path + key, buffer);
+                        }
+                    } else {
+                        fs.writeFileSync(dir_path + key, buffer);
+                    }
+                }
+            });
+        });
     }
 
-    await page.waitForTimeout(4000);
-    await page.waitForTimeout(extra_wait_ms);
+    const r = await page.goto(url, {
+        waitUntil: 'load'
+    });
 
+    await page.waitForTimeout(1000);
+    await page.waitForTimeout(extra_wait_ms);
 
     if (execute_js) {
         await page.evaluate(execute_js);
@@ -164,8 +176,6 @@ module.exports = async ({page, context}) => {
     }
 
     var html = await page.content();
-    page.close();
-
     return {
         data: {
             'content': html,
@@ -173,9 +183,8 @@ module.exports = async ({page, context}) => {
             'instock_data': instock_data,
             'screenshot': b64s,
             'status_code': r.status(),
-            'xpath_data': xpath_data,
-            'total_size': total_size
+            'xpath_data': xpath_data
         },
         type: 'application/json',
     };
-};
+};

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -17,8 +17,9 @@ function isItemInStock() {
         'as soon as stock is available',
         'ausverkauft', // sold out
         'available for back order',
-        'back-order or out of stock',
+        'awaiting stock',
         'back in stock soon',
+        'back-order or out of stock',
         'backordered',
         'benachrichtigt mich', // notify me
         'brak na stanie',
@@ -57,16 +58,20 @@ function isItemInStock() {
         'sold-out',
         'temporarily out of stock',
         'temporarily unavailable',
+        'there were no search results for',
+        'this item is currently unavailable',
         'tickets unavailable',
         'tijdelijk uitverkocht',
         'unavailable tickets',
         'vorbestellung ist bald möglich',
+        'we couldn\'t find any products that match',
         'we do not currently have an estimate of when this product will be back in stock.',
         'we don\'t know when or if this item will be back in stock.',
+        'we were not able to find a match',
         'zur zeit nicht an lager',
         '品切れ',
-        '已售完',
         '已售',
+        '已售完',
         '품절'
     ];
 
@@ -156,8 +161,6 @@ function isItemInStock() {
         const element = elementsToScan[i];
         // outside the 'fold' or some weird text in the heading area
         // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
-
-         // Should be in the "above the fold" plus about 150px
         if (element.getBoundingClientRect().top + window.scrollY >= vh + 150 || element.getBoundingClientRect().top + window.scrollY <= 100) {
             continue
         }

changedetectionio/flask_app.py

@@ -5,11 +5,11 @@ import os
 import queue
 import threading
 import time
+from .safe_jinja import render as jinja_render
+from changedetectionio.strtobool import strtobool
 from copy import deepcopy
-from distutils.util import strtobool
 from functools import wraps
 from threading import Event
-
 import flask_login
 import pytz
 import timeago
@@ -30,6 +30,7 @@ from flask_compress import Compress as FlaskCompress
 from flask_login import current_user
 from flask_paginate import Pagination, get_page_parameter
 from flask_restful import abort, Api
+from flask_cors import CORS
 from flask_wtf import CSRFProtect
 from loguru import logger
 
@@ -53,6 +54,9 @@ app = Flask(__name__,
             static_folder="static",
             template_folder="templates")
 
+# Enable CORS, especially useful for the Chrome extension to operate from anywhere
+CORS(app)
+
 # Super handy for compressing large BrowserSteps responses and others
 FlaskCompress(app)
 
@@ -315,8 +319,6 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/rss", methods=['GET'])
     def rss():
-        from jinja2 import Environment, BaseLoader
-        jinja2_env = Environment(loader=BaseLoader)
         now = time.time()
         # Always requires token set
         app_rss_token = datastore.data['settings']['application'].get('rss_access_token')
@@ -384,7 +386,7 @@ def changedetection_app(config=None, datastore_o=None):
                 # @todo Make this configurable and also consider html-colored markup
                 # @todo User could decide if <link> goes to the diff page, or to the watch link
                 rss_template = "<html><body>\n<h4><a href=\"{{watch_url}}\">{{watch_title}}</a></h4>\n<p>{{html_diff}}</p>\n</body></html>\n"
-                content = jinja2_env.from_string(rss_template).render(watch_title=watch_title, html_diff=html_diff, watch_url=watch.link)
+                content = jinja_render(template_str=rss_template, watch_title=watch_title, html_diff=html_diff, watch_url=watch.link)
 
                 fe.content(content=content, type='CDATA')
 
@@ -512,21 +514,38 @@ def changedetection_app(config=None, datastore_o=None):
 
         watch = datastore.data['watching'].get(watch_uuid) if watch_uuid else None
 
-        # validate URLS
-        if not len(request.form['notification_urls'].strip()):
-            return make_response({'error': 'No Notification URLs set'}, 400)
+        notification_urls = request.form['notification_urls'].strip().splitlines()
 
-        for server_url in request.form['notification_urls'].splitlines():
-            if len(server_url.strip()):
-                if not apobj.add(server_url):
-                    message = '{} is not a valid AppRise URL.'.format(server_url)
-                    return make_response({'error': message}, 400)
+        if not notification_urls:
+            logger.debug("Test notification - Trying by group/tag in the edit form if available")
+            # On an edit page, we should also fire off to the tags if they have notifications
+            if request.form.get('tags') and request.form['tags'].strip():
+                for k in request.form['tags'].split(','):
+                    tag = datastore.tag_exists_by_name(k.strip())
+                    notification_urls = tag.get('notifications_urls') if tag and tag.get('notifications_urls') else None
+
+        is_global_settings_form = request.args.get('mode', '') == 'global-settings'
+        is_group_settings_form = request.args.get('mode', '') == 'group-settings'
+        if not notification_urls and not is_global_settings_form and not is_group_settings_form:
+            # In the global settings, use only what is typed currently in the text box
+            logger.debug("Test notification - Trying by global system settings notifications")
+            if datastore.data['settings']['application'].get('notification_urls'):
+                notification_urls = datastore.data['settings']['application']['notification_urls']
+
+
+        if not notification_urls:
+            return 'No Notification URLs set/found'
+
+        for n_url in notification_urls:
+            if len(n_url.strip()):
+                if not apobj.add(n_url):
+                    return f'Error - {n_url} is not a valid AppRise URL.'
 
         try:
             # use the same as when it is triggered, but then override it with the form test values
             n_object = {
                 'watch_url': request.form['window_url'],
-                'notification_urls': request.form['notification_urls'].splitlines()
+                'notification_urls': notification_urls
             }
 
             # Only use if present, if not set in n_object it should use the default system value
@@ -545,7 +564,7 @@ def changedetection_app(config=None, datastore_o=None):
         except Exception as e:
             return make_response({'error': str(e)}, 400)
 
-        return 'OK'
+        return 'OK - Sent test notifications'
 
 
     @app.route("/clear_history/<string:uuid>", methods=['GET'])
@@ -582,6 +601,12 @@ def changedetection_app(config=None, datastore_o=None):
         output = render_template("clear_all_history.html")
         return output
 
+    def _watch_has_tag_options_set(watch):
+        """This should be fixed better so that Tag is some proper Model, a tag is just a Watch also"""
+        for tag_uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if tag_uuid in watch.get('tags', []) and (tag.get('include_filters') or tag.get('subtractive_selectors')):
+                return True
+
     @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
     @login_optionally_required
     # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists
@@ -752,6 +777,7 @@ def changedetection_app(config=None, datastore_o=None):
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
                                      has_empty_checktime=using_default_check_time,
                                      has_extra_headers_file=len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
+                                     has_special_tag_options=_watch_has_tag_options_set(watch=watch),
                                      is_html_webdriver=is_html_webdriver,
                                      jq_support=jq_support,
                                      playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),
@@ -1275,9 +1301,8 @@ def changedetection_app(config=None, datastore_o=None):
 
         url = request.form.get('url').strip()
         if datastore.url_exists(url):
-            flash('The URL {} already exists'.format(url), "error")
-            return redirect(url_for('index'))
-
+            flash(f'Warning, URL {url} already exists', "notice")
+            
         add_paused = request.form.get('edit_and_watch_submit_button') != None
         processor = request.form.get('processor', 'text_json_diff')
         new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
@@ -1427,6 +1452,13 @@ def changedetection_app(config=None, datastore_o=None):
                     update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
             flash("{} watches queued for rechecking".format(len(uuids)))
 
+        elif (op == 'clear-errors'):
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    datastore.data['watching'][uuid]["last_error"] = False
+            flash(f"{len(uuids)} watches errors cleared")
+
         elif (op == 'clear-history'):
             for uuid in uuids:
                 uuid = uuid.strip()

changedetectionio/forms.py

@@ -1,6 +1,6 @@
 import os
 import re
-from distutils.util import strtobool
+from changedetectionio.strtobool import strtobool
 
 from wtforms import (
     BooleanField,
@@ -236,21 +236,26 @@ class ValidateJinja2Template(object):
     def __call__(self, form, field):
         from changedetectionio import notification
 
-        from jinja2 import Environment, BaseLoader, TemplateSyntaxError, UndefinedError
+        from jinja2 import BaseLoader, TemplateSyntaxError, UndefinedError
+        from jinja2.sandbox import ImmutableSandboxedEnvironment
         from jinja2.meta import find_undeclared_variables
+        import jinja2.exceptions
 
+        # Might be a list of text, or might be just text (like from the apprise url list)
+        joined_data = ' '.join(map(str, field.data)) if isinstance(field.data, list) else f"{field.data}"
 
         try:
-            jinja2_env = Environment(loader=BaseLoader)
+            jinja2_env = ImmutableSandboxedEnvironment(loader=BaseLoader)
             jinja2_env.globals.update(notification.valid_tokens)
-
-            rendered = jinja2_env.from_string(field.data).render()
+            jinja2_env.from_string(joined_data).render()
         except TemplateSyntaxError as e:
             raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
         except UndefinedError as e:
             raise ValidationError(f"A variable or function is not defined: {e}") from e
+        except jinja2.exceptions.SecurityError as e:
+            raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
 
-        ast = jinja2_env.parse(field.data)
+        ast = jinja2_env.parse(joined_data)
         undefined = ", ".join(find_undeclared_variables(ast))
         if undefined:
             raise ValidationError(
@@ -415,7 +420,7 @@ class quickWatchForm(Form):
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
 
-    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
+    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_format = SelectField('Notification format', choices=valid_notification_formats.keys())
@@ -499,11 +504,9 @@ class watchForm(commonSettingsForm):
             result = False
 
         # Attempt to validate jinja2 templates in the URL
-        from jinja2 import Environment
-        # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
-        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
         try:
-            ready_url = str(jinja2_env.from_string(self.url.data).render())
+            from changedetectionio.safe_jinja import render as jinja_render
+            jinja_render(template_str=self.url.data)
         except Exception as e:
             self.url.errors.append('Invalid template syntax')
             result = False

changedetectionio/html_tools.py

@@ -169,14 +169,14 @@ def xpath1_filter(xpath_filter, html_content, append_pretty_line_formatting=Fals
         # And where the matched result doesn't include something that will cause Inscriptis to add a newline
         # (This way each 'match' reliably has a new-line in the diff)
         # Divs are converted to 4 whitespaces by inscriptis
-        if append_pretty_line_formatting and len(html_block) and (not hasattr( element, 'tag' ) or not element.tag in (['br', 'hr', 'div', 'p'])):
+        if append_pretty_line_formatting and len(html_block) and (not hasattr(element, 'tag') or not element.tag in (['br', 'hr', 'div', 'p'])):
             html_block += TEXT_FILTER_LIST_LINE_SUFFIX
 
-        if type(element) == etree._ElementStringResult:
-            html_block += str(element)
-        elif type(element) == etree._ElementUnicodeResult:
-            html_block += str(element)
+        # Some kind of text, UTF-8 or other
+        if isinstance(element, (str, bytes)):
+            html_block += element
         else:
+            # Return the HTML which will get parsed as text
             html_block += etree.tostring(element, pretty_print=True).decode('utf-8')
 
     return html_block

changedetectionio/model/Watch.py

@@ -1,4 +1,6 @@
-from distutils.util import strtobool
+from changedetectionio.strtobool import strtobool
+from changedetectionio.safe_jinja import render as jinja_render
+
 import os
 import re
 import time
@@ -137,12 +139,11 @@ class model(dict):
 
         ready_url = url
         if '{%' in url or '{{' in url:
-            from jinja2 import Environment
             # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
-            jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
             try:
-                ready_url = str(jinja2_env.from_string(url).render())
+                ready_url = jinja_render(template_str=url)
             except Exception as e:
+                logger.critical(f"Invalid URL template for: '{url}' - {str(e)}")
                 from flask import (
                     flash, Markup, url_for
                 )
@@ -362,6 +363,7 @@ class model(dict):
         # @todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status
         return snapshot_fname
 
+    @property
     @property
     def has_empty_checktime(self):
         # using all() + dictionary comprehension

changedetectionio/notification.py

@@ -1,6 +1,5 @@
 import apprise
 import time
-from jinja2 import Environment, BaseLoader
 from apprise import NotifyFormat
 import json
 from loguru import logger
@@ -116,6 +115,7 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
 
 def process_notification(n_object, datastore):
 
+    from .safe_jinja import render as jinja_render
     now = time.time()
     if n_object.get('notification_timestamp'):
         logger.trace(f"Time since queued {now-n_object['notification_timestamp']:.3f}s")
@@ -123,9 +123,9 @@ def process_notification(n_object, datastore):
     notification_parameters = create_notification_parameters(n_object, datastore)
 
     # Get the notification body from datastore
-    jinja2_env = Environment(loader=BaseLoader)
-    n_body = jinja2_env.from_string(n_object.get('notification_body', '')).render(**notification_parameters)
-    n_title = jinja2_env.from_string(n_object.get('notification_title', '')).render(**notification_parameters)
+    n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
+    n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
+
     n_format = valid_notification_formats.get(
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
@@ -157,7 +157,7 @@ def process_notification(n_object, datastore):
                 continue
 
             logger.info(">> Process Notification: AppRise notifying {}".format(url))
-            url = jinja2_env.from_string(url).render(**notification_parameters)
+            url = jinja_render(template_str=url, **notification_parameters)
 
             # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
             # Because different notifications may require different pre-processing, run each sequentially :(

changedetectionio/processors/__init__.py

@@ -3,7 +3,7 @@ import os
 import hashlib
 import re
 from copy import deepcopy
-from distutils.util import strtobool
+from changedetectionio.strtobool import strtobool
 from loguru import logger
 
 class difference_detection_processor():

changedetectionio/safe_jinja.py

@@ -0,0 +1,18 @@
+"""
+Safe Jinja2 render with max payload sizes
+
+See https://jinja.palletsprojects.com/en/3.1.x/sandbox/#security-considerations
+"""
+
+import jinja2.sandbox
+import typing as t
+import os
+
+JINJA2_MAX_RETURN_PAYLOAD_SIZE = 1024 * int(os.getenv("JINJA2_MAX_RETURN_PAYLOAD_SIZE_KB", 1024 * 10))
+
+
+def render(template_str, **args: t.Any) -> str:
+    jinja2_env = jinja2.sandbox.ImmutableSandboxedEnvironment(extensions=['jinja2_time.TimeExtension'])
+    output = jinja2_env.from_string(template_str).render(args)
+    return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]
+

changedetectionio/static/js/notifications.js

@@ -28,15 +28,11 @@ $(document).ready(function() {
       notification_format: $('#notification_format').val(),
       notification_title: $('#notification_title').val(),
       notification_urls: $('.notification-urls').val(),
+      tags: $('#tags').val(),
       window_url: window.location.href,
     }
 
 
-    if (!data['notification_urls'].length) {
-      alert("Notification URL list is empty, cannot send test.")
-      return;
-    }
-
     $.ajax({
       type: "POST",
       url: notification_base_url,
@@ -49,7 +45,7 @@ $(document).ready(function() {
       }
     }).done(function(data){
       console.log(data);
-      alert('Sent');
+      alert(data);
     }).fail(function(data){
       console.log(data);
       alert('There was an error communicating with the server.');

changedetectionio/static/styles/diff.css

@@ -68,7 +68,7 @@
   --color-last-checked: #bbb;
   --color-text-footer: #444;
   --color-border-watch-table-cell: #eee;
-  --color-text-watch-tag-list: #e70069;
+  --color-text-watch-tag-list: rgba(231, 0, 105, 0.4);
   --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
   --color-background-new-watch-input: var(--color-white);
   --color-text-new-watch-input: var(--color-text);
@@ -111,7 +111,7 @@ html[data-darkmode="true"] {
   --color-background-input: var(--color-grey-350);
   --color-text-input-description: var(--color-grey-600);
   --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: #fa3e92;
+  --color-text-watch-tag-list: rgba(250, 62, 146, 0.4);
   --color-background-code: var(--color-grey-200);
   --color-background-tab: rgba(0, 0, 0, 0.2);
   --color-background-tab-hover: rgba(0, 0, 0, 0.5);

changedetectionio/static/styles/scss/parts/_variables.scss

@@ -75,7 +75,7 @@
   --color-text-footer: #444;
   --color-border-watch-table-cell: #eee;
 
-  --color-text-watch-tag-list: #e70069;
+  --color-text-watch-tag-list: rgba(231, 0, 105, 0.4);
   --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
   --color-background-new-watch-input: var(--color-white);
   --color-text-new-watch-input: var(--color-text);
@@ -127,7 +127,7 @@ html[data-darkmode="true"] {
   --color-background-input: var(--color-grey-350);
   --color-text-input-description: var(--color-grey-600);
   --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: #fa3e92;
+  --color-text-watch-tag-list: rgba(250, 62, 146, 0.4);
   --color-background-code: var(--color-grey-200);
 
   --color-background-tab: rgba(0, 0, 0, 0.2);

changedetectionio/static/styles/scss/styles.scss

@@ -187,8 +187,11 @@ code {
 }
 
 .watch-tag-list {
-  color: var(--color-text-watch-tag-list);
+  color: var(--color-white);
   white-space: nowrap;
+  background: var(--color-text-watch-tag-list);
+  border-radius: 5px;
+  padding: 2px 5px;
 }
 
 .box {
@@ -1096,3 +1099,16 @@ ul {
   white-space: nowrap;
 }
 
+#chrome-extension-link {
+  img {
+    height: 21px;
+    padding: 2px;
+    vertical-align: middle;
+  }
+
+  padding: 9px;
+  border: 1px solid var(--color-grey-800);
+  border-radius: 10px;
+  vertical-align: middle;
+}
+

changedetectionio/static/styles/styles.css

@@ -284,7 +284,7 @@ ul#requests-extra_browsers {
   --color-last-checked: #bbb;
   --color-text-footer: #444;
   --color-border-watch-table-cell: #eee;
-  --color-text-watch-tag-list: #e70069;
+  --color-text-watch-tag-list: rgba(231, 0, 105, 0.4);
   --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
   --color-background-new-watch-input: var(--color-white);
   --color-text-new-watch-input: var(--color-text);
@@ -327,7 +327,7 @@ html[data-darkmode="true"] {
   --color-background-input: var(--color-grey-350);
   --color-text-input-description: var(--color-grey-600);
   --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: #fa3e92;
+  --color-text-watch-tag-list: rgba(250, 62, 146, 0.4);
   --color-background-code: var(--color-grey-200);
   --color-background-tab: rgba(0, 0, 0, 0.2);
   --color-background-tab-hover: rgba(0, 0, 0, 0.5);
@@ -532,8 +532,11 @@ code {
     margin: 0 3px 0 5px; }
 
 .watch-tag-list {
-  color: var(--color-text-watch-tag-list);
-  white-space: nowrap; }
+  color: var(--color-white);
+  white-space: nowrap;
+  background: var(--color-text-watch-tag-list);
+  border-radius: 5px;
+  padding: 2px 5px; }
 
 .box {
   max-width: 80%;
@@ -1180,3 +1183,13 @@ ul {
   .restock-label.not-in-stock {
     background-color: var(--color-background-button-cancel);
     color: #777; }
+
+#chrome-extension-link {
+  padding: 9px;
+  border: 1px solid var(--color-grey-800);
+  border-radius: 10px;
+  vertical-align: middle; }
+  #chrome-extension-link img {
+    height: 21px;
+    padding: 2px;
+    vertical-align: middle; }

changedetectionio/store.py

@@ -1,4 +1,4 @@
-from distutils.util import strtobool
+from changedetectionio.strtobool import strtobool
 
 from flask import (
     flash
@@ -657,7 +657,10 @@ class ChangeDetectionStore:
         return res
 
     def tag_exists_by_name(self, tag_name):
-        return any(v.get('title', '').lower() == tag_name.lower() for k, v in self.__data['settings']['application']['tags'].items())
+        # Check if any tag dictionary has a 'title' attribute matching the provided tag_name
+        tags = self.__data['settings']['application']['tags'].values()
+        return next((v for v in tags if v.get('title', '').lower() == tag_name.lower()),
+                    None)
 
     def get_updates_available(self):
         import inspect

changedetectionio/strtobool.py

@@ -0,0 +1,23 @@
+# Because strtobool was removed in python 3.12 distutils
+
+_MAP = {
+    'y': True,
+    'yes': True,
+    't': True,
+    'true': True,
+    'on': True,
+    '1': True,
+    'n': False,
+    'no': False,
+    'f': False,
+    'false': False,
+    'off': False,
+    '0': False
+}
+
+
+def strtobool(value):
+    try:
+        return _MAP[str(value).lower()]
+    except KeyError:
+        raise ValueError('"{}" is not a valid bool value'.format(value))

changedetectionio/templates/base.html

@@ -147,7 +147,19 @@
     <section class="content">
         <div id="overlay">
             <div class="content">
-                <strong>changedetection.io needs your support!</strong><br>
+                <h4>Try our Chrome extension</h4>
+                <p>
+                    <a id="chrome-extension-link"
+                       title="Try our new Chrome Extension!"
+                       href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
+                        <img src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}">
+                        Chrome Webstore
+                    </a>
+                </p>
+
+                Easily add the current web-page from your browser directly into your changedetection.io tool, more great features coming soon!
+
+                <h4>Changedetection.io needs your support!</h4>
                 <p>
                     You can help us by supporting changedetection.io on these platforms;
                 </p>

changedetectionio/templates/edit.html

@@ -7,7 +7,8 @@
 <script>
     const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
-    const browser_steps_fetch_screenshot_image_url="{{url_for('browser_steps.browser_steps_fetch_screenshot_image', uuid=uuid)}}";
+    <!-- Should be _external so that firefox and others load it more reliably -->
+    const browser_steps_fetch_screenshot_image_url="{{url_for('browser_steps.browser_steps_fetch_screenshot_image', uuid=uuid, _external=True)}}";
     const browser_steps_last_error_step={{ watch.browser_steps_last_error_step|tojson }};
     const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
     const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
@@ -31,6 +32,7 @@
 <script src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
 {% endif %}
 
+{% set has_tag_filters_extra="WARNING: Watch has tag/groups set with special filters\n" if has_special_tag_options else '' %}
 <script src="{{url_for('static_content', group='js', filename='recheck-proxy.js')}}" defer></script>
 
 <div class="edit-form monospaced-textarea">
@@ -280,7 +282,7 @@ User-Agent: wonderbra 1.0") }}
                     <div class="pure-control-group">
                         {% set field = render_field(form.include_filters,
                             rows=5,
-                            placeholder="#example
+                            placeholder=has_tag_filters_extra+"#example
 xpath://body/div/span[contains(@class, 'example-class')]",
                             class="m-d")
                         %}
@@ -316,13 +318,14 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                 </span>
                     </div>
                 <fieldset class="pure-control-group">
-                    {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
+                    {{ render_field(form.subtractive_selectors, rows=5, placeholder=has_tag_filters_extra+"header
 footer
 nav
 .stockticker") }}
                     <span class="pure-form-message-inline">
                         <ul>
                           <li> Remove HTML element(s) by CSS selector before text conversion. </li>
+                          <li> Don't paste HTML here, use only CSS selectors </li>
                           <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
                         </ul>
                       </span>
@@ -436,7 +439,7 @@ Unavailable") }}
                     <div class="pure-control-group">
                         {% if visualselector_enabled %}
                             <span class="pure-form-message-inline">
-                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection &dash; after the <i>Browser Steps</i> has completed.<br><br>
+                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection &dash; after the <i>Browser Steps</i> has completed, this tool is a helper to manage filters in the  "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab.
                             </span>
 
                             <div id="selector-header">

changedetectionio/templates/import.html

@@ -107,7 +107,7 @@
                                     <option value="" style="color: #aaa"> -- none --</option>
                                     <option value="url">URL</option>
                                     <option value="title">Title</option>
-                                    <option value="include_filter">CSS/xPath filter</option>
+                                    <option value="include_filters">CSS/xPath filter</option>
                                     <option value="tag">Group / Tag name(s)</option>
                                     <option value="interval_minutes">Recheck time (minutes)</option>
                                 </select></td>

changedetectionio/templates/settings.html

@@ -4,7 +4,7 @@
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test', mode="global-settings")}}";
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
 {% endif %}
@@ -168,12 +168,12 @@ nav
            </div>
 
             <div class="tab-pane-inner" id="api">
-
+                <h4>API Access</h4>
                 <p>Drive your changedetection.io via API, More about <a href="https://github.com/dgtlmoon/changedetection.io/wiki/API-Reference">API access here</a></p>
 
                 <div class="pure-control-group">
                     {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
-                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header</div><br>
+                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header - required for the Chrome Extension to work</div><br>
                     <div class="pure-form-message-inline"><br>API Key <span id="api-key">{{api_key}}</span>
                         <span style="display:none;" id="api-key-copy" >copy</span>
                     </div>
@@ -181,6 +181,20 @@ nav
                 <div class="pure-control-group">
                     <a href="{{url_for('settings_reset_api_key')}}" class="pure-button button-small button-cancel">Regenerate API key</a>
                 </div>
+                <div class="pure-control-group">
+                    <h4>Chrome Extension</h4>
+                    <p>Easily add any web-page to your changedetection.io installation from within Chrome.</p>
+                    <strong>Step 1</strong> Install the extension, <strong>Step 2</strong> Navigate to this page,
+                    <strong>Step 3</strong> Open the extension from the toolbar and click "<i>Sync API Access</i>"
+                    <p>
+                        <a id="chrome-extension-link"
+                           title="Try our new Chrome Extension!"
+                           href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
+                            <img src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}">
+                            Chrome Webstore
+                        </a>
+                    </p>
+                </div>
             </div>
             <div class="tab-pane-inner" id="proxies">
                 <div id="recommended-proxy">

changedetectionio/templates/watch-overview.html

@@ -37,6 +37,7 @@
         <button class="pure-button button-secondary button-xsmall" name="op" value="assign-tag" id="checkbox-assign-tag">Tag</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="mark-viewed">Mark viewed</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="notification-default">Use default notification</button>
+        <button class="pure-button button-secondary button-xsmall" name="op" value="clear-errors">Clear errors</button>
         <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="clear-history">Clear/reset history</button>
         <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="delete">Delete</button>
     </div>
@@ -168,7 +169,7 @@
                 <td>
                     <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                        class="recheck pure-button pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
-                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button pure-button-primary">Edit</a>
+                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
                     {% if watch.history_n >= 2 %}
 
                         {%  if is_unviewed %}

changedetectionio/tests/restock/test_restock.py

@@ -95,7 +95,7 @@ def test_restock_detection(client, live_server):
 
     # We should have a notification
     time.sleep(2)
-    assert os.path.isfile("test-datastore/notification.txt")
+    assert os.path.isfile("test-datastore/notification.txt"), "Notification received"
     os.unlink("test-datastore/notification.txt")
 
     # Default behaviour is to only fire notification when it goes OUT OF STOCK -> IN STOCK
@@ -103,4 +103,9 @@ def test_restock_detection(client, live_server):
     set_original_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    assert not os.path.isfile("test-datastore/notification.txt")
+    assert not os.path.isfile("test-datastore/notification.txt"), "No notification should have fired when it went OUT OF STOCK by default"
+
+    # BUT we should see that it correctly shows "not in stock"
+    res = client.get(url_for("index"))
+    assert b'not-in-stock' in res.data, "Correctly showing NOT IN STOCK in the list after it changed from IN STOCK"
+

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python3
-
+import os.path
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
@@ -107,7 +107,6 @@ def test_check_add_line_contains_trigger(client, live_server):
     #live_server_setup(live_server)
 
     # Give the endpoint time to spin up
-    time.sleep(1)
     test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://') + "?xxx={{ watch_url }}"
 
     res = client.post(
@@ -166,6 +165,7 @@ def test_check_add_line_contains_trigger(client, live_server):
 
     # Takes a moment for apprise to fire
     time.sleep(3)
+    assert os.path.isfile("test-datastore/notification.txt"), "Notification fired because I can see the output file"
     with open("test-datastore/notification.txt", 'r') as f:
         response= f.read()
         assert '-Oh yes please-' in response

changedetectionio/tests/test_group.py

@@ -100,6 +100,12 @@ def test_setup_group_tag(client, live_server):
     assert b'Should be only this' in res.data
     assert b'And never this' not in res.data
 
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+    # 2307 the UI notice should appear in the placeholder
+    assert b'WARNING: Watch has tag/groups set with special filters' in res.data
 
     # RSS Group tag filter
     # An extra one that should be excluded

changedetectionio/tests/test_jinja2.py

@@ -2,15 +2,15 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 
 
+def test_setup(client, live_server):
+    live_server_setup(live_server)
+
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
 def test_jinja2_in_url_query(client, live_server):
-    live_server_setup(live_server)
-
-    # Give the endpoint time to spin up
-    time.sleep(1)
+    #live_server_setup(live_server)
 
     # Add our URL to the import page
     test_url = url_for('test_return_query', _external=True)
@@ -24,10 +24,35 @@ def test_jinja2_in_url_query(client, live_server):
         follow_redirects=True
     )
     assert b"Watch added" in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
+
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True
     )
     assert b'date=2' in res.data
+
+# https://techtonics.medium.com/secure-templating-with-jinja2-understanding-ssti-and-jinja2-sandbox-environment-b956edd60456
+def test_jinja2_security_url_query(client, live_server):
+    #live_server_setup(live_server)
+
+    # Add our URL to the import page
+    test_url = url_for('test_return_query', _external=True)
+
+    # because url_for() will URL-encode the var, but we dont here
+    full_url = "{}?{}".format(test_url,
+                              "date={{ ''.__class__.__mro__[1].__subclasses__()}}", )
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": full_url, "tags": "test"},
+        follow_redirects=True
+    )
+    assert b"Watch added" in res.data
+    wait_for_all_checks(client)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'is invalid and cannot be used' in res.data
+    # Some of the spewed output from the subclasses
+    assert b'dict_values' not in res.data

changedetectionio/tests/test_xpath_selector.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+# -*- coding: utf-8 -*-
 
 import time
 from flask import url_for
@@ -255,6 +255,69 @@ def test_xpath23_prefix_validation(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
+def test_xpath1_lxml(client, live_server):
+    #live_server_setup(live_server)
+
+    d = '''<?xml version="1.0" encoding="UTF-8"?>
+    <rss xmlns:taxo="http://purl.org/rss/1.0/modules/taxonomy/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
+    	<channel>
+    		<title>rpilocator.com</title>
+    		<link>https://rpilocator.com</link>
+    		<description>Find Raspberry Pi Computers in Stock</description>
+    		<lastBuildDate>Thu, 19 May 2022 23:27:30 GMT</lastBuildDate>
+    		<image>
+    			<url>https://rpilocator.com/favicon.png</url>
+    			<title>rpilocator.com</title>
+    			<link>https://rpilocator.com/</link>
+    			<width>32</width>
+    			<height>32</height>
+    		</image>
+    		<item>
+    			<title>Stock Alert (UK): RPi CM4</title>
+    			<foo>something else unrelated</foo>
+    		</item>
+    		<item>
+    			<title>Stock Alert (UK): Big monitorěěěě</title>
+    			<foo>something else unrelated</foo>
+    		</item>		
+    	</channel>
+    </rss>'''.encode('utf-8')
+
+    with open("test-datastore/endpoint-content.txt", "wb") as f:
+        f.write(d)
+
+
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"include_filters": "xpath1://title/text()", "url": test_url, "tags": "", "headers": "",
+              'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    ##### #2312
+    wait_for_all_checks(client)
+    res = client.get(url_for("index"))
+    assert b'_ElementStringResult' not in res.data # tested with 5.1.1 when it was removed and 5.1.0
+    assert b'Exception' not in res.data
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"rpilocator.com" in res.data  # in selector
+    assert "Stock Alert (UK): Big monitorěěěě".encode('utf-8') in res.data  # not in selector
+
+    #####
+
 
 def test_xpath1_validation(client, live_server):
     # Add our URL to the import page

changedetectionio/tests/unit/test_jinja2_security.py

@@ -0,0 +1,57 @@
+#!/usr/bin/python3
+
+# run from dir above changedetectionio/ dir
+# python3 -m unittest changedetectionio.tests.unit.test_jinja2_security
+
+import unittest
+from changedetectionio import safe_jinja
+
+
+# mostly
+class TestJinja2SSTI(unittest.TestCase):
+
+    def test_exception(self):
+        import jinja2
+
+        # Where sandbox should kick in
+        attempt_list = [
+            "My name is {{ self.__init__.__globals__.__builtins__.__import__('os').system('id') }}",
+            "{{ self._TemplateReference__context.cycler.__init__.__globals__.os }}",
+            "{{ self.__init__.__globals__.__builtins__.__import__('os').popen('id').read() }}",
+            "{{cycler.__init__.__globals__.os.popen('id').read()}}",
+            "{{joiner.__init__.__globals__.os.popen('id').read()}}",
+            "{{namespace.__init__.__globals__.os.popen('id').read()}}",
+            "{{ ''.__class__.__mro__[2].__subclasses__()[40]('/tmp/hello.txt', 'w').write('Hello here !') }}",
+            "My name is {{ self.__init__.__globals__ }}",
+            "{{ dict.__base__.__subclasses__() }}"
+        ]
+        for attempt in attempt_list:
+            with self.assertRaises(jinja2.exceptions.SecurityError):
+                safe_jinja.render(attempt)
+
+    def test_exception_debug_calls(self):
+        import jinja2
+        # Where sandbox should kick in - configs and debug calls
+        attempt_list = [
+            "{% debug %}",
+        ]
+        for attempt in attempt_list:
+            # Usually should be something like 'Encountered unknown tag 'debug'.'
+            with self.assertRaises(jinja2.exceptions.TemplateSyntaxError):
+                safe_jinja.render(attempt)
+
+    # https://book.hacktricks.xyz/pentesting-web/ssti-server-side-template-injection/jinja2-ssti#accessing-global-objects
+    def test_exception_empty_calls(self):
+        import jinja2
+        attempt_list = [
+            "{{config}}",
+            "{{ debug }}"
+            "{{[].__class__}}",
+        ]
+        for attempt in attempt_list:
+            self.assertEqual(len(safe_jinja.render(attempt)), 0, f"string test '{attempt}' is correctly empty")
+
+
+
+if __name__ == '__main__':
+    unittest.main()

changedetectionio/tests/util.py

@@ -116,7 +116,7 @@ def extract_UUID_from_client(client):
     )
     # <span id="api-key">{{api_key}}</span>
 
-    m = re.search('edit/(.+?)"', str(res.data))
+    m = re.search('edit/(.+?)[#"]', str(res.data))
     uuid = m.group(1)
     return uuid.strip()
 

changedetectionio/update_worker.py

@@ -462,7 +462,7 @@ class update_worker(threading.Thread):
                     except Exception as e:
                         logger.error(f"Exception reached processing watch UUID: {uuid}")
                         logger.error(str(e))
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Exception: " + str(e)})
                         # Other serious error
                         process_changedetection_results = False
 #                        import traceback

requirements.txt

@@ -9,6 +9,7 @@ flask-login>=0.6.3
 flask-paginate
 flask_expects_json~=1.7
 flask_restful
+flask_cors # For the Chrome extension to operate
 flask_wtf~=1.2
 flask~=2.3
 inscriptis~=2.2
@@ -35,10 +36,12 @@ dnspython==2.3.0 # related to eventlet fixes
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise~=1.7.1
+apprise~=1.7.4
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
-paho-mqtt
+# and 2.0.0 https://github.com/dgtlmoon/changedetection.io/issues/2241 not yet compatible
+# use v1.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
+paho-mqtt < 2.0.0
 
 # This mainly affects some ARM builds, which unlike the other builds ignores "ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1"
 # so without this pinning, the newer versions on ARM will forcefully try to build rust, which results in "rust compiler not found"
@@ -49,7 +52,7 @@ cryptography~=3.4
 beautifulsoup4
 
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
-lxml
+lxml >=4.8.0,<6
 
 # XPath 2.0-3.1 support - 4.2.0 broke something?
 elementpath==4.1.5
@@ -72,7 +75,7 @@ pillow
 # playwright is installed at Dockerfile build time because it's not available on all platforms
 
 # experimental release
-pyppeteer-ng==2.0.0rc2
+pyppeteer-ng==2.0.0rc5
 
 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup
 pytest ~=7.2