Re #3891 - Handle inline favicons

* feat: adding missing Czech translation strings

* feat: adding more missing Czech translation strings

* feat: adding more missing Czech translation strings

* feat: adding more missing Czech translation strings

.github/workflows/test-stack-reusable-workflow.yml

@@ -99,11 +99,7 @@ jobs:
 
       - name: Run Unit Tests
         run: |
-          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
-          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
-          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
-          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
-          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_html_to_text'         
+          docker run test-changedetectionio bash -c 'cd changedetectionio;pytest tests/unit/'
 
   # Basic pytest tests with ancillary services
   basic-tests:
@@ -587,6 +583,10 @@ jobs:
         run: |
           docker run  -e EXTRA_PACKAGES=changedetection.io-osint-processor test-changedetectionio bash -c 'cd changedetectionio;pytest -vvv -s  tests/plugins/test_processor.py::test_check_plugin_processor'
 
+      - name: Plugin get_html_head_extras hook injects into base.html
+        run: |
+          docker run test-changedetectionio bash -c 'cd changedetectionio;pytest -vvv -s tests/plugins/test_html_head_extras.py'
+
   # Container startup tests
   container-tests:
     runs-on: ubuntu-latest

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 # Semver means never use .01, or 00. Should be .1.
-__version__ = '0.54.6'
+__version__ = '0.54.8'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/blueprint/backups/__init__.py

@@ -98,8 +98,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     backups_blueprint.register_blueprint(construct_restore_blueprint(datastore))
     backup_threads = []
 
-    @login_optionally_required
     @backups_blueprint.route("/request-backup", methods=['GET'])
+    @login_optionally_required
     def request_backup():
         if any(thread.is_alive() for thread in backup_threads):
             flash(gettext("A backup is already running, check back in a few minutes"), "error")
@@ -141,8 +141,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         return backup_info
 
-    @login_optionally_required
     @backups_blueprint.route("/download/<string:filename>", methods=['GET'])
+    @login_optionally_required
     def download_backup(filename):
         import re
         filename = filename.strip()
@@ -165,9 +165,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         logger.debug(f"Backup download request for '{full_path}'")
         return send_from_directory(os.path.abspath(datastore.datastore_path), filename, as_attachment=True)
 
-    @login_optionally_required
     @backups_blueprint.route("/", methods=['GET'])
     @backups_blueprint.route("/create", methods=['GET'])
+    @login_optionally_required
     def create():
         backups = find_backups()
         output = render_template("backup_create.html",
@@ -176,8 +176,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                                  )
         return output
 
-    @login_optionally_required
     @backups_blueprint.route("/remove-backups", methods=['GET'])
+    @login_optionally_required
     def remove_backups():
 
         backup_filepath = os.path.join(datastore.datastore_path, BACKUP_FILENAME_FORMAT.format("*"))

changedetectionio/blueprint/backups/restore.py

@@ -174,8 +174,8 @@ def construct_restore_blueprint(datastore):
     restore_blueprint = Blueprint('restore', __name__, template_folder="templates")
     restore_threads = []
 
-    @login_optionally_required
     @restore_blueprint.route("/restore", methods=['GET'])
+    @login_optionally_required
     def restore():
         form = RestoreForm()
         return render_template("backup_restore.html",
@@ -184,8 +184,8 @@ def construct_restore_blueprint(datastore):
                                max_upload_mb=_MAX_UPLOAD_BYTES // (1024 * 1024),
                                max_decompressed_mb=_MAX_DECOMPRESSED_BYTES // (1024 * 1024))
 
-    @login_optionally_required
     @restore_blueprint.route("/restore/start", methods=['POST'])
+    @login_optionally_required
     def backups_restore_start():
         if any(t.is_alive() for t in restore_threads):
             flash(gettext("A restore is already running, check back in a few minutes"), "error")

changedetectionio/blueprint/backups/templates/backup_restore.html

@@ -20,8 +20,7 @@
                 <p>{{ _('Restore a backup. Must be a .zip backup file created on/after v0.53.1 (new database layout).') }}</p>
                 <p>{{ _('Note: This does not override the main application settings, only watches and groups.') }}</p>
                 <p class="pure-form-message">
-                    {{ _('Max upload size: %(upload)s MB &nbsp;·&nbsp; Max decompressed size: %(decomp)s MB',
-                         upload=max_upload_mb, decomp=max_decompressed_mb) }}
+                    {{ _('Max upload size: %(upload)s MB, Max decompressed size: %(decomp)s MB', upload=max_upload_mb, decomp=max_decompressed_mb) }}
                 </p>
 
                 <form class="pure-form pure-form-stacked settings"

changedetectionio/blueprint/browser_steps/__init__.py

@@ -268,8 +268,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         return browsersteps_start_session
 
 
-    @login_optionally_required
     @browser_steps_blueprint.route("/browsersteps_start_session", methods=['GET'])
+    @login_optionally_required
     def browsersteps_start_session():
         # A new session was requested, return sessionID
         import uuid
@@ -304,8 +304,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         logger.debug("Starting connection with playwright - done")
         return {'browsersteps_session_id': browsersteps_session_id}
 
-    @login_optionally_required
     @browser_steps_blueprint.route("/browsersteps_image", methods=['GET'])
+    @login_optionally_required
     def browser_steps_fetch_screenshot_image():
         from flask import (
             make_response,
@@ -330,8 +330,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             return make_response('Unable to fetch image, is the URL correct? does the watch exist? does the step_type-n.jpeg exist?', 401)
 
     # A request for an action was received
-    @login_optionally_required
     @browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])
+    @login_optionally_required
     def browsersteps_ui_update():
         import base64
 

changedetectionio/blueprint/imports/importer.py

@@ -160,8 +160,7 @@ class import_xlsx_wachete(Importer):
             flash(gettext("Unable to read export XLSX file, something wrong with the file?"), 'error')
             return
 
-        row_id = 2
-        for row in wb.active.iter_rows(min_row=row_id):
+        for row_id, row in enumerate(wb.active.iter_rows(min_row=2), start=2):
             try:
                 extras = {}
                 data = {}
@@ -212,8 +211,6 @@ class import_xlsx_wachete(Importer):
             except Exception as e:
                 logger.error(e)
                 flash(gettext("Error processing row number {}, check all cell data types are correct, row was skipped.").format(row_id), 'error')
-            else:
-                row_id += 1
 
         flash(gettext("{} imported from Wachete .xlsx in {:.2f}s").format(len(self.new_uuids), time.time() - now))
 
@@ -241,10 +238,10 @@ class import_xlsx_custom(Importer):
 
         # @todo cehck atleast 2 rows, same in other method
         from changedetectionio.forms import validate_url
-        row_i = 1
+        row_i = 0
 
         try:
-            for row in wb.active.iter_rows():
+            for row_i, row in enumerate(wb.active.iter_rows(), start=1):
                 url = None
                 tags = None
                 extras = {}
@@ -295,7 +292,5 @@ class import_xlsx_custom(Importer):
         except Exception as e:
             logger.error(e)
             flash(gettext("Error processing row number {}, check all cell data types are correct, row was skipped.").format(row_i), 'error')
-        else:
-            row_i += 1
 
         flash(gettext("{} imported from custom .xlsx in {:.2f}s").format(len(self.new_uuids), time.time() - now))

changedetectionio/blueprint/imports/templates/import.html

@@ -9,6 +9,7 @@
             <li class="tab" id=""><a href="#url-list">{{ _('URL List') }}</a></li>
             <li class="tab"><a href="#distill-io">{{ _('Distill.io') }}</a></li>
             <li class="tab"><a href="#xlsx">{{ _('.XLSX & Wachete') }}</a></li>
+            <li class="tab"><a href="{{url_for('backups.restore.restore')}}">{{ _('Backup Restore') }}</a></li>
         </ul>
     </div>
 

changedetectionio/blueprint/tags/__init__.py

@@ -22,11 +22,14 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         tag_count = Counter(tag for watch in datastore.data['watching'].values() if watch.get('tags') for tag in watch['tags'])
 
+        from changedetectionio import processors
         output = render_template("groups-overview.html",
                                  app_rss_token=datastore.data['settings']['application'].get('rss_access_token'),
                                  available_tags=sorted_tags,
                                  form=add_form,
+                                 generate_tag_colors=processors.generate_processor_badge_colors,
                                  tag_count=tag_count,
+                                 wcag_text_color=processors.wcag_text_color,
                                  )
 
         return output
@@ -208,9 +211,17 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             template = env.from_string(template_str)
             included_content = template.render(**template_args)
 
+        # Watches whose URL currently matches this tag's pattern
+        matching_watches = {
+            w_uuid: watch
+            for w_uuid, watch in datastore.data['watching'].items()
+            if default.matches_url(watch.get('url', ''))
+        }
+
         output = render_template("edit-tag.html",
                                  extra_form_content=included_content,
                                  extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None,
+                                 matching_watches=matching_watches,
                                  settings_application=datastore.data['settings']['application'],
                                  **template_args
                                  )

changedetectionio/blueprint/tags/form.py

@@ -10,12 +10,11 @@ from changedetectionio.processors.restock_diff.forms import processor_settings_f
 
 class group_restock_settings_form(restock_settings_form):
     overrides_watch = BooleanField('Activate for individual watches in this tag/group?', default=False)
+    url_match_pattern = StringField('Auto-apply to watches with URLs matching',
+                                    render_kw={"placeholder": "e.g. *://example.com/* or github.com/myorg"})
+    tag_colour = StringField('Tag colour', default='')
 
 class SingleTag(Form):
 
     name = StringField('Tag name', [validators.InputRequired()], render_kw={"placeholder": "Name"})
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
-
-
-
-

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -43,6 +43,46 @@
                     <div class="pure-control-group">
                         {{ render_field(form.title, placeholder="https://...", required=true, class="m-d") }}
                     </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.url_match_pattern, class="m-d") }}
+                        <span class="pure-form-message-inline">{{ _('Automatically applies this tag to any watch whose URL matches. Supports wildcards: <code>*example.com*</code> or plain substring: <code>github.com/myorg</code>')|safe }}</span>
+                    </div>
+                    {% if matching_watches %}
+                    <div class="pure-control-group">
+                        <label>{{ _('Currently matching watches') }} ({{ matching_watches|length }})</label>
+                        <ul class="tag-url-match-list">
+                            {% for w_uuid, w in matching_watches.items() %}
+                            <li><a href="{{ url_for('ui.ui_edit.edit_page', uuid=w_uuid) }}">{{ w.label }}</a></li>
+                            {% endfor %}
+                        </ul>
+                    </div>
+                    {% endif %}
+                    <div class="pure-control-group">
+                        <label>{{ _('Tag colour') }}</label>
+                        <div style="display:flex; align-items:center; gap:0.75em;">
+                            <input type="checkbox" id="use_custom_colour"
+                                   {% if data.get('tag_colour') %}checked{% endif %}>
+                            <label for="use_custom_colour" style="margin:0">{{ _('Custom colour') }}</label>
+                            <input type="color" id="tag_colour_picker"
+                                   value="{{ data.get('tag_colour') or '#4f8ef7' }}"
+                                   {% if not data.get('tag_colour') %}disabled{% endif %}>
+                            <input type="hidden" name="tag_colour" id="tag_colour_hidden"
+                                   value="{{ data.get('tag_colour', '') }}">
+                        </div>
+                        <span class="pure-form-message-inline">{{ _('Leave unchecked to use the auto-generated colour based on the tag name.') }}</span>
+                    </div>
+                    <script>
+                    (function () {
+                        var cb = document.getElementById('use_custom_colour');
+                        var picker = document.getElementById('tag_colour_picker');
+                        var hidden = document.getElementById('tag_colour_hidden');
+                        picker.addEventListener('input', function () { hidden.value = this.value; });
+                        cb.addEventListener('change', function () {
+                            picker.disabled = !this.checked;
+                            hidden.value = this.checked ? picker.value : '';
+                        });
+                    })();
+                    </script>
                 </fieldset>
             </div>
 

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -3,6 +3,26 @@
 {% from '_helpers.html' import render_simple_field, render_field %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='modal.js')}}"></script>
+<style>
+{%- for uuid, tag in available_tags -%}
+{%- if tag and tag.title -%}
+{%- set class_name = tag.title|sanitize_tag_class -%}
+{%- if tag.get('tag_colour') -%}
+.watch-tag-list.tag-{{ class_name }} { background-color: {{ tag.tag_colour }}; color: {{ wcag_text_color(tag.tag_colour) }}; }
+{%- else -%}
+{%- set colors = generate_tag_colors(tag.title) -%}
+.watch-tag-list.tag-{{ class_name }} {
+  background-color: {{ colors['light']['bg'] }};
+  color: {{ colors['light']['color'] }};
+}
+html[data-darkmode="true"] .watch-tag-list.tag-{{ class_name }} {
+  background-color: {{ colors['dark']['bg'] }};
+  color: {{ colors['dark']['color'] }};
+}
+{%- endif -%}
+{%- endif -%}
+{%- endfor -%}
+</style>
 
 <div class="box">
     <form class="pure-form" action="{{ url_for('tags.form_tag_add') }}" method="POST" id="new-watch-form">
@@ -48,7 +68,7 @@
                     <a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
                 </td>
                 <td>{{ "{:,}".format(tag_count[uuid]) if uuid in tag_count else 0 }}</td>
-                <td class="title-col inline"> <a href="{{url_for('watchlist.index', tag=uuid) }}">{{ tag.title }}</a></td>
+                <td class="title-col inline"> <a href="{{url_for('watchlist.index', tag=uuid) }}" class="watch-tag-list tag-{{ tag.title|sanitize_tag_class }}">{{ tag.title }}</a></td>
                 <td>
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">{{ _('Edit') }}</a>
                     <a href="{{ url_for('ui.form_watch_checknow', tag=uuid) }}" class="pure-button pure-button-primary" >{{ _('Recheck') }}</a>

changedetectionio/blueprint/ui/edit.py

@@ -320,7 +320,12 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                 'using_global_webdriver_wait': not default['webdriver_delay'],
                 'uuid': uuid,
                 'watch': watch,
-                'capabilities': capabilities
+                'capabilities': capabilities,
+                'auto_applied_tags': {
+                    tag_uuid: tag
+                    for tag_uuid, tag in datastore.data['settings']['application']['tags'].items()
+                    if tag_uuid not in watch.get('tags', []) and tag.matches_url(watch.get('url', ''))
+                },
             }
 
             included_content = None

changedetectionio/blueprint/ui/templates/edit.html

@@ -81,6 +81,14 @@
                     <div class="pure-control-group">
                         {{ render_field(form.tags) }}
                         <span class="pure-form-message-inline">{{ _('Organisational tag/group name used in the main listing page') }}</span>
+                        {% if auto_applied_tags %}
+                        <span class="pure-form-message-inline">
+                            {{ _('Also automatically applied by URL pattern:') }}
+                            {% for tag_uuid, tag in auto_applied_tags.items() %}
+                            <a href="{{ url_for('tags.form_tag_edit', uuid=tag_uuid) }}" class="watch-tag-list tag-{{ tag.title|sanitize_tag_class }}">{{ tag.title }}</a>
+                            {% endfor %}
+                        </span>
+                        {% endif %}
                     </div>
                     <div class="pure-control-group inline-radio">
                         {{ render_field(form.processor) }}

changedetectionio/blueprint/watchlist/__init__.py

@@ -92,6 +92,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             extra_classes='has-queue' if not update_q.empty() else '',
             form=form,
             generate_tag_colors=processors.generate_processor_badge_colors,
+            wcag_text_color=processors.wcag_text_color,
             guid=datastore.data['app_guid'],
             has_proxies=proxy_list,
             hosted_sticky=os.getenv("SALTED_PASS", False) == False,

changedetectionio/blueprint/watchlist/templates/watch-overview.html

@@ -71,6 +71,13 @@ document.addEventListener('DOMContentLoaded', function() {
 {%- for uuid, tag in tags -%}
 {%- if tag and tag.title -%}
 {%- set class_name = tag.title|sanitize_tag_class -%}
+{%- if tag.get('tag_colour') -%}
+.button-tag.tag-{{ class_name }},
+.watch-tag-list.tag-{{ class_name }} {
+  background-color: {{ tag.tag_colour }};
+  color: {{ wcag_text_color(tag.tag_colour) }};
+}
+{%- else -%}
 {%- set colors = generate_tag_colors(tag.title) -%}
 .button-tag.tag-{{ class_name }} {
   background-color: {{ colors['light']['bg'] }};
@@ -92,6 +99,7 @@ html[data-darkmode="true"] .watch-tag-list.tag-{{ class_name }} {
   color: {{ colors['dark']['color'] }};
 }
 {%- endif -%}
+{%- endif -%}
 {%- endfor -%}
 </style>
 <div class="box" id="form-quick-watch-add">

changedetectionio/content_fetchers/playwright.py

@@ -49,6 +49,9 @@ async def capture_full_page_async(page, screenshot_format='JPEG', watch_uuid=Non
     if page_height > page.viewport_size['height']:
         if page_height < step_size:
             step_size = page_height # Incase page is bigger than default viewport but smaller than proposed step size
+        # Never set viewport taller than our max capture height - otherwise one screenshot chunk
+        # captures the whole (e.g. 8098px) page even when SCREENSHOT_MAX_HEIGHT=1000
+        step_size = min(step_size, SCREENSHOT_MAX_TOTAL_HEIGHT)
         viewport_start = time.time()
         logger.debug(f"{watch_info}Setting bigger viewport to step through large page width W{page.viewport_size['width']}xH{step_size} because page_height > viewport_size")
         # Set viewport to a larger size to capture more content at once

changedetectionio/content_fetchers/puppeteer.py

@@ -75,6 +75,9 @@ async def capture_full_page(page, screenshot_format='JPEG', watch_uuid=None, loc
     if page_height > page.viewport['height']:
         if page_height < step_size:
             step_size = page_height # Incase page is bigger than default viewport but smaller than proposed step size
+        # Never set viewport taller than our max capture height - otherwise one screenshot chunk
+        # captures the whole page even when SCREENSHOT_MAX_HEIGHT is set smaller
+        step_size = min(step_size, SCREENSHOT_MAX_TOTAL_HEIGHT)
         viewport_start = time.time()
         await page.setViewport({'width': page.viewport['width'], 'height': step_size})
         viewport_time = time.time() - viewport_start

changedetectionio/content_fetchers/res/favicon-fetcher.js

@@ -38,26 +38,39 @@
       if (a.size !== b.size) {
         return b.size - a.size;
       }
-      
+
       // Second priority: apple-touch-icon over regular icon
       const isAppleA = /apple-touch-icon/.test(a.rel);
       const isAppleB = /apple-touch-icon/.test(b.rel);
       if (isAppleA && !isAppleB) return -1;
       if (!isAppleA && isAppleB) return 1;
-      
+
       // Third priority: icons with no size attribute (fallback icons) last
       const hasNoSizeA = !a.hasSizes;
       const hasNoSizeB = !b.hasSizes;
       if (hasNoSizeA && !hasNoSizeB) return 1;
       if (!hasNoSizeA && hasNoSizeB) return -1;
-      
+
       return 0;
     });
 
     const timeoutMs = 2000;
+    // 1 MB — matches the server-side limit in bump_favicon()
+    const MAX_BYTES = 1 * 1024 * 1024;
 
     for (const icon of icons) {
       try {
+        // Inline data URI — no network fetch needed, data is already here
+        if (icon.href.startsWith('data:')) {
+          const match = icon.href.match(/^data:([^;]+);base64,([A-Za-z0-9+/=]+)$/);
+          if (!match) continue;
+          const mime_type = match[1];
+          const base64 = match[2];
+          // Rough size check: base64 is ~4/3 the binary size
+          if (base64.length * 0.75 > MAX_BYTES) continue;
+          return { url: icon.href, mime_type, base64 };
+        }
+
         const controller = new AbortController();
         const timeout = setTimeout(() => controller.abort(), timeoutMs);
 
@@ -74,12 +87,15 @@
 
         const blob = await resp.blob();
 
+        if (blob.size > MAX_BYTES) continue;
+
         // Convert blob to base64
         const reader = new FileReader();
         return await new Promise(resolve => {
           reader.onloadend = () => {
             resolve({
               url: icon.href,
+              mime_type: blob.type,
               base64: reader.result.split(",")[1]
             });
           };
@@ -98,4 +114,3 @@
   // Auto-execute and return result for page.evaluate()
   return await window.getFaviconAsBlob();
 })();
-

changedetectionio/content_fetchers/screenshot_handler.py

@@ -56,6 +56,10 @@ def stitch_images_worker_raw_bytes(pipe_conn, original_page_height, capture_heig
             im.close()
         del images
 
+        # Clip stitched image to capture_height (chunks may overshoot by up to step_size-1 px)
+        if total_height > capture_height:
+            stitched = stitched.crop((0, 0, max_width, capture_height))
+
         # Draw caption only if page was trimmed
         if original_page_height > capture_height:
             draw = ImageDraw.Draw(stitched)

changedetectionio/content_fetchers/webdriver_selenium.py

@@ -104,15 +104,17 @@ class fetcher(Fetcher):
 
             from selenium.webdriver.remote.remote_connection import RemoteConnection
             from selenium.webdriver.remote.webdriver import WebDriver as RemoteWebDriver
+            from selenium.webdriver.remote.client_config import ClientConfig
+            from urllib3.util import Timeout
             driver = None
             try:
-                # Create the RemoteConnection and set timeout (e.g., 30 seconds)
-                remote_connection = RemoteConnection(
-                    self.browser_connection_url,
+                connection_timeout = int(os.getenv("WEBDRIVER_CONNECTION_TIMEOUT", 90))
+                client_config = ClientConfig(
+                    remote_server_addr=self.browser_connection_url,
+                    timeout=Timeout(connect=connection_timeout, total=connection_timeout)
                 )
-                remote_connection.set_timeout(30)  # seconds
+                remote_connection = RemoteConnection(client_config=client_config)
 
-                # Now create the driver with the RemoteConnection
                 driver = RemoteWebDriver(
                     command_executor=remote_connection,
                     options=options

changedetectionio/diff/__init__.py

@@ -45,8 +45,38 @@ CHANGED_INTO_PLACEMARKER_CLOSED = '@changed_into_PLACEMARKER_CLOSED'
 # Compiled regex patterns for performance
 WHITESPACE_NORMALIZE_RE = re.compile(r'\s+')
 
+# Regexes built from the constants above — no brittle hardcoded strings
+_EXTRACT_REMOVED_RE = re.compile(
+    re.escape(REMOVED_PLACEMARKER_OPEN) + r'(.*?)' + re.escape(REMOVED_PLACEMARKER_CLOSED)
+    + r'|' +
+    re.escape(CHANGED_PLACEMARKER_OPEN) + r'(.*?)' + re.escape(CHANGED_PLACEMARKER_CLOSED)
+)
+_EXTRACT_ADDED_RE = re.compile(
+    re.escape(ADDED_PLACEMARKER_OPEN) + r'(.*?)' + re.escape(ADDED_PLACEMARKER_CLOSED)
+    + r'|' +
+    re.escape(CHANGED_INTO_PLACEMARKER_OPEN) + r'(.*?)' + re.escape(CHANGED_INTO_PLACEMARKER_CLOSED)
+)
 
-def render_inline_word_diff(before_line: str, after_line: str, ignore_junk: bool = False, markdown_style: str = None, tokenizer: str = 'words_and_html') -> tuple[str, bool]:
+
+def extract_changed_from(raw_diff: str) -> str:
+    """Extract only the removed/changed-from fragments from a raw diff string.
+
+    Useful for {{diff_changed_from}} — gives just the old value (e.g. old price),
+    not the full surrounding line. Multiple fragments joined with newlines.
+    """
+    return '\n'.join(m.group(1) or m.group(2) for m in _EXTRACT_REMOVED_RE.finditer(raw_diff))
+
+
+def extract_changed_to(raw_diff: str) -> str:
+    """Extract only the added/changed-into fragments from a raw diff string.
+
+    Useful for {{diff_changed_to}} — gives just the new value (e.g. new price),
+    not the full surrounding line. Multiple fragments joined with newlines.
+    """
+    return '\n'.join(m.group(1) or m.group(2) for m in _EXTRACT_ADDED_RE.finditer(raw_diff))
+
+
+def render_inline_word_diff(before_line: str, after_line: str, ignore_junk: bool = False, markdown_style: str = None, tokenizer: str = 'words_and_html', include_change_type_prefix: bool = True) -> tuple[str, bool]:
     """
     Render word-level differences between two lines inline using diff-match-patch library.
 
@@ -133,14 +163,20 @@ def render_inline_word_diff(before_line: str, after_line: str, ignore_junk: bool
         if removed_tokens:
             removed_full = ''.join(removed_tokens).rstrip()
             trailing_removed = ''.join(removed_tokens)[len(removed_full):] if len(''.join(removed_tokens)) > len(removed_full) else ''
-            result_parts.append(f'{CHANGED_PLACEMARKER_OPEN}{removed_full}{CHANGED_PLACEMARKER_CLOSED}{trailing_removed}')
+            if include_change_type_prefix:
+                result_parts.append(f'{CHANGED_PLACEMARKER_OPEN}{removed_full}{CHANGED_PLACEMARKER_CLOSED}{trailing_removed}')
+            else:
+                result_parts.append(f'{removed_full}{trailing_removed}')
 
         if added_tokens:
             if result_parts:  # Add newline between removed and added
                 result_parts.append('\n')
             added_full = ''.join(added_tokens).rstrip()
             trailing_added = ''.join(added_tokens)[len(added_full):] if len(''.join(added_tokens)) > len(added_full) else ''
-            result_parts.append(f'{CHANGED_INTO_PLACEMARKER_OPEN}{added_full}{CHANGED_INTO_PLACEMARKER_CLOSED}{trailing_added}')
+            if include_change_type_prefix:
+                result_parts.append(f'{CHANGED_INTO_PLACEMARKER_OPEN}{added_full}{CHANGED_INTO_PLACEMARKER_CLOSED}{trailing_added}')
+            else:
+                result_parts.append(f'{added_full}{trailing_added}')
 
         return ''.join(result_parts), has_changes
     else:
@@ -150,21 +186,27 @@ def render_inline_word_diff(before_line: str, after_line: str, ignore_junk: bool
             if op == 0:  # Equal
                 result_parts.append(text)
             elif op == 1:  # Insertion
-                # Don't wrap empty content (e.g., whitespace-only tokens after rstrip)
-                content = text.rstrip()
-                trailing = text[len(content):] if len(text) > len(content) else ''
-                if content:
-                    result_parts.append(f'{ADDED_PLACEMARKER_OPEN}{content}{ADDED_PLACEMARKER_CLOSED}{trailing}')
+                if not include_change_type_prefix:
+                    result_parts.append(text)
                 else:
-                    result_parts.append(trailing)
+                    # Don't wrap empty content (e.g., whitespace-only tokens after rstrip)
+                    content = text.rstrip()
+                    trailing = text[len(content):] if len(text) > len(content) else ''
+                    if content:
+                        result_parts.append(f'{ADDED_PLACEMARKER_OPEN}{content}{ADDED_PLACEMARKER_CLOSED}{trailing}')
+                    else:
+                        result_parts.append(trailing)
             elif op == -1:  # Deletion
-                # Don't wrap empty content (e.g., whitespace-only tokens after rstrip)
-                content = text.rstrip()
-                trailing = text[len(content):] if len(text) > len(content) else ''
-                if content:
-                    result_parts.append(f'{REMOVED_PLACEMARKER_OPEN}{content}{REMOVED_PLACEMARKER_CLOSED}{trailing}')
+                if not include_change_type_prefix:
+                    result_parts.append(text)
                 else:
-                    result_parts.append(trailing)
+                    # Don't wrap empty content (e.g., whitespace-only tokens after rstrip)
+                    content = text.rstrip()
+                    trailing = text[len(content):] if len(text) > len(content) else ''
+                    if content:
+                        result_parts.append(f'{REMOVED_PLACEMARKER_OPEN}{content}{REMOVED_PLACEMARKER_CLOSED}{trailing}')
+                    else:
+                        result_parts.append(trailing)
 
         return ''.join(result_parts), has_changes
 
@@ -360,7 +402,7 @@ def customSequenceMatcher(
 
             # Use inline word-level diff for single line replacements when word_diff is enabled
             if word_diff and len(before_lines) == 1 and len(after_lines) == 1:
-                inline_diff, has_changes = render_inline_word_diff(before_lines[0], after_lines[0], ignore_junk=ignore_junk, tokenizer=tokenizer)
+                inline_diff, has_changes = render_inline_word_diff(before_lines[0], after_lines[0], ignore_junk=ignore_junk, tokenizer=tokenizer, include_change_type_prefix=include_change_type_prefix)
                 # Check if there are any actual changes (not just whitespace when ignore_junk is enabled)
                 if ignore_junk and not has_changes:
                     # No real changes, skip this line

changedetectionio/flask_app.py

@@ -212,6 +212,11 @@ def _is_safe_valid_url(test_url):
     from .validate_url import is_safe_valid_url
     return is_safe_valid_url(test_url)
 
+@app.template_global('get_html_head_extras')
+def _get_html_head_extras():
+    from .pluggy_interface import collect_html_head_extras
+    return collect_html_head_extras()
+
 
 @app.template_filter('format_number_locale')
 def _jinja2_filter_format_number_locale(value: float) -> str:

changedetectionio/forms.py

@@ -667,9 +667,11 @@ class ValidateCSSJSONXPATHInput(object):
                     # `jq` requires full compilation in windows and so isn't generally available
                     raise ValidationError("jq not support not found")
 
+                from changedetectionio.html_tools import validate_jq_expression
                 input = line.replace('jq:', '')
 
                 try:
+                    validate_jq_expression(input)
                     jq.compile(input)
                 except (ValueError) as e:
                     message = field.gettext('\'%s\' is not a valid jq expression. (%s)')
@@ -723,7 +725,7 @@ class ValidateStartsWithRegex(object):
                 raise ValidationError(self.message or _l("Invalid value."))
 
 class quickWatchForm(Form):
-    url = fields.URLField(_l('URL'), validators=[validateURL()])
+    url = StringField(_l('URL'), validators=[validateURL()])
     tags = StringTagUUID(_l('Group tag'), validators=[validators.Optional()])
     watch_submit_button = SubmitField(_l('Watch'), render_kw={"class": "pure-button pure-button-primary"})
     processor = RadioField(_l('Processor'), choices=lambda: processors.available_processors(), default=processors.get_default_processor)
@@ -777,7 +779,7 @@ class SingleBrowserStep(Form):
 
 class processor_text_json_diff_form(commonSettingsForm):
 
-    url = fields.URLField('Web Page URL', validators=[validateURL()])
+    url = StringField('Web Page URL', validators=[validateURL()])
     tags = StringTagUUID('Group Tag', [validators.Optional()], default='')
 
     time_between_check = EnhancedFormField(
@@ -1005,7 +1007,7 @@ class globalSettingsApplicationForm(commonSettingsForm):
         render_kw={"placeholder": "0.1", "style": "width: 8em;"}
     )
 
-    password = SaltyPasswordField(_l('Password'))
+    password = SaltyPasswordField(_l('Password'), render_kw={"autocomplete": "new-password"})
     pager_size = IntegerField(_l('Pager size'),
                               render_kw={"style": "width: 5em;"},
                               validators=[validators.NumberRange(min=0,

changedetectionio/html_tools.py

@@ -4,6 +4,7 @@ from loguru import logger
 from typing import List
 import html
 import json
+import os
 import re
 
 # HTML added to be sure each result matching a filter (.example) gets converted to a new line by Inscriptis
@@ -13,6 +14,45 @@ PERL_STYLE_REGEX = r'^/(.*?)/([a-z]*)?$'
 
 TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.I | re.S)
 META_CS  = re.compile(r'<meta[^>]+charset=["\']?\s*([a-z0-9_\-:+.]+)', re.I)
+
+# jq builtins that can leak sensitive data or cause harm when user-supplied expressions are executed.
+# env/$ENV reads all process environment variables (passwords, API keys, etc.)
+# include/import can read arbitrary files from disk
+# input/inputs reads beyond the supplied JSON data
+# debug/stderr leaks data to stderr
+# halt/halt_error terminates the process (DoS)
+_JQ_BLOCKED_PATTERNS = [
+    (re.compile(r'\benv\b'),                    'env (reads environment variables)'),
+    (re.compile(r'\$ENV\b'),                    '$ENV (reads environment variables)'),
+    (re.compile(r'\binclude\b'),                'include (reads files from disk)'),
+    (re.compile(r'\bimport\b'),                 'import (reads files from disk)'),
+    (re.compile(r'\binputs?\b'),                'input/inputs (reads beyond provided data)'),
+    (re.compile(r'\bdebug\b'),                  'debug (leaks data to stderr)'),
+    (re.compile(r'\bstderr\b'),                 'stderr (leaks data to stderr)'),
+    (re.compile(r'\bhalt(?:_error)?\b'),        'halt/halt_error (terminates the process)'),
+    (re.compile(r'\$__loc__\b'),                '$__loc__ (leaks file path information)'),
+    (re.compile(r'\bbuiltins\b'),               'builtins (enumerates available functions)'),
+    (re.compile(r'\bmodulemeta\b'),             'modulemeta (leaks module information)'),
+    (re.compile(r'\$JQ_BUILD_CONFIGURATION\b'), '$JQ_BUILD_CONFIGURATION (leaks build information)'),
+]
+
+def validate_jq_expression(expression: str) -> None:
+    """Raise ValueError if the jq expression uses any dangerous builtin.
+
+    User-supplied jq expressions are executed server-side. Without this check,
+    builtins like `env` expose every process environment variable (SALTED_PASS,
+    proxy credentials, API keys, etc.) as watch output.
+    """
+    from changedetectionio.strtobool import strtobool
+    if strtobool(os.getenv('JQ_ALLOW_RISKY_EXPRESSIONS', 'false')):
+        return
+
+    for pattern, description in _JQ_BLOCKED_PATTERNS:
+        if pattern.search(expression):
+            msg = f"jq expression uses disallowed builtin: {description}"
+            logger.critical(f"Security: blocked jq expression containing '{description}' - expression: {expression!r}")
+            raise ValueError(msg)
+
 META_CT  = re.compile(r'<meta[^>]+http-equiv=["\']?content-type["\']?[^>]*content=["\'][^>]*charset=([a-z0-9_\-:+.]+)', re.I)
 
 # 'price' , 'lowPrice', 'highPrice' are usually under here
@@ -30,6 +70,12 @@ _DEFAULT_UNSAFE_XPATH3_FUNCTIONS = [
     'unparsed-text-available',
     'doc',
     'doc-available',
+    'json-doc',
+    'json-doc-available',
+    'collection',           # XPath 2.0+: loads XML node collections from arbitrary URIs
+    'uri-collection',       # XPath 3.0+: enumerates URIs from resource collections
+    'transform',            # XPath 3.1: XSLT transformation (currently raises, block proactively)
+    'load-xquery-module',   # XPath 3.1: loads XQuery modules (currently raises, block proactively)
     'environment-variable',
     'available-environment-variables',
 ]
@@ -378,12 +424,16 @@ def _parse_json(json_data, json_filter):
             raise Exception("jq not support not found")
 
         if json_filter.startswith("jq:"):
-            jq_expression = jq.compile(json_filter.removeprefix("jq:"))
+            expr = json_filter.removeprefix("jq:")
+            validate_jq_expression(expr)
+            jq_expression = jq.compile(expr)
             match = jq_expression.input(json_data).all()
             return _get_stripped_text_from_json_match(match)
 
         if json_filter.startswith("jqraw:"):
-            jq_expression = jq.compile(json_filter.removeprefix("jqraw:"))
+            expr = json_filter.removeprefix("jqraw:")
+            validate_jq_expression(expr)
+            jq_expression = jq.compile(expr)
             match = jq_expression.input(json_data).all()
             return '\n'.join(str(item) for item in match)
 

changedetectionio/languages.py

@@ -28,18 +28,20 @@ def get_timeago_locale(flask_locale):
         str: timeago library locale code (e.g., 'en', 'zh_CN', 'pt_PT')
     """
     locale_map = {
-        'zh': 'zh_CN',      # Chinese Simplified
+        'zh': 'zh_CN',          # Chinese Simplified
         # timeago library just hasn't been updated to use the more modern locale naming convention, before BCP 47 / RFC 5646.
-        'zh_TW': 'zh_TW',   # Chinese Traditional (timeago uses zh_TW)
+        'zh_TW': 'zh_TW',       # Chinese Traditional (timeago uses zh_TW)
         'zh_Hant_TW': 'zh_TW',  # Flask-Babel normalizes zh_TW to zh_Hant_TW, map back to timeago's zh_TW
-        'pt': 'pt_PT',      # Portuguese (Portugal)
-        'sv': 'sv_SE',      # Swedish
-        'no': 'nb_NO',      # Norwegian Bokmål
-        'hi': 'in_HI',      # Hindi
-        'cs': 'en',         # Czech not supported by timeago, fallback to English
-        'uk': 'uk',         # Ukrainian
-        'en_GB': 'en',      # British English - timeago uses 'en'
-        'en_US': 'en',      # American English - timeago uses 'en'
+        'pt': 'pt_PT',          # Portuguese (Portugal)
+        'pt_BR': 'pt_BR',       # Portuguese (Brasil)
+        'sv': 'sv_SE',          # Swedish
+        'no': 'nb_NO',          # Norwegian Bokmål
+        'hi': 'in_HI',          # Hindi
+        'cs': 'en',             # Czech not supported by timeago, fallback to English
+        'ja': 'ja',             # Japanese
+        'uk': 'uk',             # Ukrainian
+        'en_GB': 'en',          # British English - timeago uses 'en'
+        'en_US': 'en',          # American English - timeago uses 'en'
     }
     return locale_map.get(flask_locale, flask_locale)
 
@@ -53,7 +55,8 @@ LANGUAGE_DATA = {
     'ko': {'flag': 'fi fi-kr fis', 'name': '한국어'},
     'cs': {'flag': 'fi fi-cz fis', 'name': 'Čeština'},
     'es': {'flag': 'fi fi-es fis', 'name': 'Español'},
-    'pt': {'flag': 'fi fi-pt fis', 'name': 'Português'},
+    'pt': {'flag': 'fi fi-pt fis', 'name': 'Português (Portugal)'},
+    'pt_BR': {'flag': 'fi fi-br fis', 'name': 'Português (Brasil)'},
     'it': {'flag': 'fi fi-it fis', 'name': 'Italiano'},
     'ja': {'flag': 'fi fi-jp fis', 'name': '日本語'},
     'zh': {'flag': 'fi fi-cn fis', 'name': '中文 (简体)'},

changedetectionio/model/Tag.py

@@ -46,11 +46,26 @@ class model(EntityPersistenceMixin, watch_base):
         super(model, self).__init__(*arg, **kw)
 
         self['overrides_watch'] = kw.get('default', {}).get('overrides_watch')
+        self['url_match_pattern'] = kw.get('default', {}).get('url_match_pattern', '')
 
         if kw.get('default'):
             self.update(kw['default'])
             del kw['default']
 
+    def matches_url(self, url: str) -> bool:
+        """Return True if this tag should be auto-applied to the given watch URL.
+
+        Wildcard patterns (*,?,[ ) use fnmatch; anything else is a case-insensitive
+        substring match. Returns False if no pattern is configured.
+        """
+        import fnmatch
+        pattern = self.get('url_match_pattern', '').strip()
+        if not pattern or not url:
+            return False
+        if any(c in pattern for c in ('*', '?', '[')):
+            return fnmatch.fnmatch(url.lower(), pattern.lower())
+        return pattern.lower() in url.lower()
+
     # _save_to_disk() method provided by EntityPersistenceMixin
     # commit() and _get_commit_data() methods inherited from watch_base
     # Tag uses default _get_commit_data() (includes all keys)

changedetectionio/model/Watch.py

@@ -798,24 +798,50 @@ class model(EntityPersistenceMixin, watch_base):
         # Also in the case that the file didnt exist
         return True
 
-    def bump_favicon(self, url, favicon_base_64: str) -> None:
+    def bump_favicon(self, url, favicon_base_64: str, mime_type: str = None) -> None:
         from urllib.parse import urlparse
         import base64
         import binascii
-        decoded = None
+        import re
 
-        if url:
+        MAX_FAVICON_BYTES = 1 * 1024 * 1024  # 1 MB
+
+        MIME_TO_EXT = {
+            'image/png': 'png',
+            'image/x-icon': 'ico',
+            'image/vnd.microsoft.icon': 'ico',
+            'image/jpeg': 'jpg',
+            'image/gif': 'gif',
+            'image/svg+xml': 'svg',
+            'image/webp': 'webp',
+            'image/bmp': 'bmp',
+        }
+
+        extension = None
+
+        # If the caller already resolved the MIME type (e.g. from blob.type or a data URI),
+        # use that directly — it's more reliable than guessing from a URL path.
+        if mime_type:
+            extension = MIME_TO_EXT.get(mime_type.lower().split(';')[0].strip(), None)
+
+        # Fall back to extracting extension from URL path, unless it's a data URI.
+        if not extension and url and not url.startswith('data:'):
             try:
                 parsed = urlparse(url)
                 filename = os.path.basename(parsed.path)
-                (base, extension) = filename.lower().strip().rsplit('.', 1)
+                (_base, ext) = filename.lower().strip().rsplit('.', 1)
+                extension = ext
             except ValueError:
-                logger.error(f"UUID: {self.get('uuid')} Cant work out file extension from '{url}'")
-                return None
-        else:
-            # Assume favicon.ico
-            base = "favicon"
-            extension = "ico"
+                logger.warning(f"UUID: {self.get('uuid')} Cant work out file extension from '{url}', defaulting to ico")
+
+        # Handle data URIs: extract MIME type from the URI itself when not already known
+        if not extension and url and url.startswith('data:'):
+            m = re.match(r'^data:([^;]+);base64,', url)
+            if m:
+                extension = MIME_TO_EXT.get(m.group(1).lower(), None)
+
+        if not extension:
+            extension = 'ico'
 
         fname = os.path.join(self.data_dir, f"favicon.{extension}")
 
@@ -824,22 +850,27 @@ class model(EntityPersistenceMixin, watch_base):
             decoded = base64.b64decode(favicon_base_64, validate=True)
         except (binascii.Error, ValueError) as e:
             logger.warning(f"UUID: {self.get('uuid')} FavIcon save data (Base64) corrupt? {str(e)}")
-        else:
-            if decoded:
-                try:
-                    with open(fname, 'wb') as f:
-                        f.write(decoded)
+            return None
 
-                    # Invalidate module-level favicon filename cache for this watch
-                    _FAVICON_FILENAME_CACHE.pop(self.data_dir, None)
+        if len(decoded) > MAX_FAVICON_BYTES:
+            logger.warning(f"UUID: {self.get('uuid')} Favicon too large ({len(decoded)} bytes), skipping")
+            return None
 
-                    # A signal that could trigger the socket server to update the browser also
-                    watch_check_update = signal('watch_favicon_bump')
-                    if watch_check_update:
-                        watch_check_update.send(watch_uuid=self.get('uuid'))
+        try:
+            with open(fname, 'wb') as f:
+                f.write(decoded)
 
-                except Exception as e:
-                    logger.warning(f"UUID: {self.get('uuid')} error saving FavIcon to {fname} - {str(e)}")
+            # Invalidate module-level favicon filename cache for this watch
+            _FAVICON_FILENAME_CACHE.pop(self.data_dir, None)
+
+            # A signal that could trigger the socket server to update the browser also
+            watch_check_update = signal('watch_favicon_bump')
+            if watch_check_update:
+                watch_check_update.send(watch_uuid=self.get('uuid'))
+
+        except Exception as e:
+            logger.warning(f"UUID: {self.get('uuid')} error saving FavIcon to {fname} - {str(e)}")
+            return None
 
         # @todo - Store some checksum and only write when its different
         logger.debug(f"UUID: {self.get('uuid')} updated favicon to at {fname}")

changedetectionio/notification_service.py

@@ -88,6 +88,28 @@ class FormattableTimestamp(str):
             return self._dt.isoformat()
 
 
+class FormattableExtract(str):
+    """
+    A str subclass that holds only the extracted changed fragments from a diff.
+    Used for {{diff_changed_from}} and {{diff_changed_to}} tokens.
+
+        {{ diff_changed_from }}   → old value(s) only, e.g. "$99.99"
+        {{ diff_changed_to }}     → new value(s) only, e.g. "$109.99"
+
+    Multiple changed fragments are joined with newlines.
+    Being a str subclass means it is natively JSON serializable.
+    """
+    def __new__(cls, prev_snapshot, current_snapshot, extract_fn):
+        if prev_snapshot or current_snapshot:
+            from changedetectionio import diff as diff_module
+            raw = diff_module.render_diff(prev_snapshot, current_snapshot, word_diff=True)
+            extracted = extract_fn(raw)
+        else:
+            extracted = ''
+        instance = super().__new__(cls, extracted)
+        return instance
+
+
 class FormattableDiff(str):
     """
     A str subclass representing a rendered diff. As a plain string it renders
@@ -161,6 +183,8 @@ class NotificationContextData(dict):
             'diff_patch': FormattableDiff('', '', patch_format=True),
             'diff_removed': FormattableDiff('', '', include_added=False),
             'diff_removed_clean': FormattableDiff('', '', include_added=False, include_change_type_prefix=False),
+            'diff_changed_from': FormattableExtract('', '', extract_fn=lambda x: x),
+            'diff_changed_to': FormattableExtract('', '', extract_fn=lambda x: x),
             'diff_url': None,
             'markup_text_links_to_html_links': False, # If automatic conversion of plaintext to HTML should happen
             'notification_timestamp': time.time(),
@@ -244,16 +268,27 @@ def add_rendered_diff_to_notification_vars(notification_scan_text:str, prev_snap
         'diff_removed_clean': {'word_diff': word_diff, 'include_added': False, 'include_change_type_prefix': False},
     }
 
+    from changedetectionio.diff import extract_changed_from, extract_changed_to
+    extract_specs = {
+        'diff_changed_from': extract_changed_from,
+        'diff_changed_to':   extract_changed_to,
+    }
+
     ret = {}
     rendered_count = 0
-    # Only create FormattableDiff objects for diff keys actually used in the notification text
+    # Only create FormattableDiff/FormattableExtract objects for diff keys actually used in the notification text
     for key in NotificationContextData().keys():
-        if key.startswith('diff') and key in diff_specs:
-            # Check if this placeholder is actually used in the notification text
-            pattern = rf"(?<![A-Za-z0-9_]){re.escape(key)}(?![A-Za-z0-9_])"
-            if re.search(pattern, notification_scan_text, re.IGNORECASE):
-                ret[key] = FormattableDiff(prev_snapshot, current_snapshot, **diff_specs[key])
-                rendered_count += 1
+        if not key.startswith('diff'):
+            continue
+        pattern = rf"(?<![A-Za-z0-9_]){re.escape(key)}(?![A-Za-z0-9_])"
+        if not re.search(pattern, notification_scan_text, re.IGNORECASE):
+            continue
+        if key in diff_specs:
+            ret[key] = FormattableDiff(prev_snapshot, current_snapshot, **diff_specs[key])
+            rendered_count += 1
+        elif key in extract_specs:
+            ret[key] = FormattableExtract(prev_snapshot, current_snapshot, extract_fn=extract_specs[key])
+            rendered_count += 1
 
     if rendered_count:
         logger.trace(f"Rendered {rendered_count} diff placeholder(s) {sorted(ret.keys())} in {time.time() - now:.3f}s")
@@ -461,7 +496,7 @@ Thanks - Your omniscient changedetection.io installation.
         n_object = NotificationContextData({
             'notification_title': f"Changedetection.io - Alert - Browser step at position {step} could not be run",
             'notification_body': body,
-            'notification_format': self._check_cascading_vars('notification_format', watch),
+            'notification_format': _check_cascading_vars(self.datastore, 'notification_format', watch),
         })
         n_object['markup_text_links_to_html_links'] = n_object.get('notification_format').startswith('html')
 

changedetectionio/pluggy_interface.py

@@ -174,6 +174,64 @@ class ChangeDetectionSpec:
         """
         pass
 
+    @hookspec
+    def get_html_head_extras():
+        """Return HTML to inject into the <head> of every page via base.html.
+
+        Plugins can use this to add <script>, <style>, or <link> tags that should
+        be present on all pages.  Return a raw HTML string or None.
+
+        IMPORTANT: Always use Flask's url_for() for any src/href URLs so that
+        sub-path deployments (nginx reverse proxy with USE_X_SETTINGS / X-Forwarded-Prefix)
+        work correctly.  This hook is called inside a request context so url_for() is
+        always available.
+
+        For small amounts of CSS/JS, return them inline — no file-serving needed::
+
+            from changedetectionio.pluggy_interface import hookimpl
+
+            @hookimpl
+            def get_html_head_extras(self):
+                return (
+                    '<style>.my-module-banner { color: red; }</style>\\n'
+                    '<script>console.log("my_module_content loaded");</script>'
+                )
+
+        For larger assets, register your own lightweight Flask routes in the plugin
+        module and point to them with url_for() so the sub-path prefix is handled
+        automatically::
+
+            from flask import url_for, Response
+            from changedetectionio.pluggy_interface import hookimpl
+            from changedetectionio.flask_app import app as _app
+
+            MY_CSS = ".my-module-example { color: red; }"
+            MY_JS  = "console.log('my_module_content loaded');"
+
+            @_app.route('/my_module_content/css')
+            def my_module_content_css():
+                return Response(MY_CSS, mimetype='text/css',
+                                headers={'Cache-Control': 'max-age=3600'})
+
+            @_app.route('/my_module_content/js')
+            def my_module_content_js():
+                return Response(MY_JS, mimetype='application/javascript',
+                                headers={'Cache-Control': 'max-age=3600'})
+
+            @hookimpl
+            def get_html_head_extras(self):
+                css = url_for('my_module_content_css')
+                js  = url_for('my_module_content_js')
+                return (
+                    f'<link rel="stylesheet" href="{css}">\\n'
+                    f'<script src="{js}" defer></script>'
+                )
+
+        Returns:
+            str or None: Raw HTML string to inject inside <head>, or None
+        """
+        pass
+
 
 # Set up Plugin Manager
 plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE)
@@ -606,4 +664,20 @@ def apply_update_finalize(update_handler, watch, datastore, processing_exception
     except Exception as e:
         # Don't let plugin errors crash the worker
         logger.error(f"Error in update_finalize hook: {e}")
-        logger.exception(f"update_finalize hook exception details:")
+        logger.exception(f"update_finalize hook exception details:")
+
+
+def collect_html_head_extras():
+    """Collect and combine HTML head extras from all plugins.
+
+    Called from a Flask template global so it always runs inside a request context.
+    This means url_for() works correctly in plugin implementations, including when the
+    app is deployed under a sub-path via USE_X_SETTINGS / X-Forwarded-Prefix (ProxyFix
+    sets SCRIPT_NAME so url_for() automatically prepends the prefix).
+
+    Returns:
+        str: Combined HTML string to inject inside <head>, or empty string
+    """
+    results = plugin_manager.hook.get_html_head_extras()
+    parts = [r for r in results if r]
+    return "\n".join(parts) if parts else ""

changedetectionio/processors/__init__.py

@@ -341,6 +341,18 @@ def get_processor_descriptions():
     return descriptions
 
 
+def wcag_text_color(hex_bg: str) -> str:
+    """Return #000000 or #ffffff for maximum WCAG contrast against hex_bg."""
+    hex_bg = hex_bg.lstrip('#')
+    if len(hex_bg) != 6:
+        return '#000000'
+    r, g, b = (int(hex_bg[i:i+2], 16) / 255 for i in (0, 2, 4))
+    def lin(c):
+        return c / 12.92 if c <= 0.04045 else ((c + 0.055) / 1.055) ** 2.4
+    L = 0.2126 * lin(r) + 0.7152 * lin(g) + 0.0722 * lin(b)
+    return '#000000' if L > 0.179 else '#ffffff'
+
+
 def generate_processor_badge_colors(processor_name):
     """
     Generate consistent colors for a processor badge based on its name.

changedetectionio/store/__init__.py

@@ -980,12 +980,20 @@ class ChangeDetectionStore(DatastoreUpdatesMixin, FileSavingDataStore):
     def get_all_tags_for_watch(self, uuid):
         """This should be in Watch model but Watch doesn't have access to datastore, not sure how to solve that yet"""
         watch = self.data['watching'].get(uuid)
+        if not watch:
+            return {}
 
-        # Should return a dict of full tag info linked by UUID
-        if watch:
-            return dictfilt(self.__data['settings']['application']['tags'], watch.get('tags', []))
+        # Start with manually assigned tags
+        result = dictfilt(self.__data['settings']['application']['tags'], watch.get('tags', []))
 
-        return {}
+        # Additionally include any tag whose url_match_pattern matches this watch's URL
+        watch_url = watch.get('url', '')
+        if watch_url:
+            for tag_uuid, tag in self.__data['settings']['application']['tags'].items():
+                if tag_uuid not in result and tag.matches_url(watch_url):
+                    result[tag_uuid] = tag
+
+        return result
 
     @property
     def extra_browsers(self):

changedetectionio/templates/_common_fields.html

@@ -98,6 +98,14 @@
                 <td><code>{{ '{{diff_patch}}' }}</code></td>
                 <td>{{ _('The diff output - patch in unified format') }}</td>
             </tr>
+            <tr>
+                <td><code>{{ '{{diff_changed_from}}' }}</code></td>
+                <td>{{ _('Only the changed words/values from the previous version — e.g. the old price. Best when a single value changes per line; multiple changed fragments are joined by newline.') }}</td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{diff_changed_to}}' }}</code></td>
+                <td>{{ _('Only the changed words/values from the new version — e.g. the new price. Best when a single value changes per line; multiple changed fragments are joined by newline.') }}</td>
+            </tr>
             <tr>
                 <td><code>{{ '{{current_snapshot}}' }}</code></td>
                 <td>{{ _('The current snapshot text contents value, useful when combined with JSON or CSS filters') }}

changedetectionio/templates/base.html

@@ -45,6 +45,10 @@
     <script src="{{url_for('static_content', group='js', filename='socket.io.min.js')}}"></script>
     <script src="{{url_for('static_content', group='js', filename='realtime.js')}}" defer></script>
     {% endif %}
+    {%- set _html_head_extras = get_html_head_extras() -%}
+    {%- if _html_head_extras %}
+    {{ _html_head_extras | safe }}
+    {%- endif %}
   </head>
 
   <body class="{{extra_classes}}">

changedetectionio/tests/plugins/test_html_head_extras.py

@@ -0,0 +1,83 @@
+"""Test that plugins can inject HTML into base.html <head> via get_html_head_extras hookimpl."""
+import pytest
+from flask import url_for, Response
+
+from changedetectionio.pluggy_interface import hookimpl, plugin_manager
+
+_MY_JS = "console.log('my_module_content loaded');"
+_MY_CSS = ".my-module-example { color: red; }"
+
+
+class _HeadExtrasPlugin:
+    """Test plugin that injects tags pointing at its own Flask routes."""
+
+    @hookimpl
+    def get_html_head_extras(self):
+        css_url = url_for('test_plugin_my_module_content_css')
+        js_url  = url_for('test_plugin_my_module_content_js')
+        return (
+            f'<link rel="stylesheet" id="test-head-extra-css" href="{css_url}">\n'
+            f'<script id="test-head-extra-js" src="{js_url}" defer></script>'
+        )
+
+
+@pytest.fixture(scope='module')
+def plugin_routes(live_server):
+    """Register plugin asset routes once per module (Flask routes can't be added twice)."""
+    app = live_server.app
+
+    @app.route('/test-plugin/my_module_content/css')
+    def test_plugin_my_module_content_css():
+        return Response(_MY_CSS, mimetype='text/css',
+                        headers={'Cache-Control': 'max-age=3600'})
+
+    @app.route('/test-plugin/my_module_content/js')
+    def test_plugin_my_module_content_js():
+        return Response(_MY_JS, mimetype='application/javascript',
+                        headers={'Cache-Control': 'max-age=3600'})
+
+
+@pytest.fixture
+def head_extras_plugin(plugin_routes):
+    """Register the hookimpl for one test then unregister it — function-scoped for clean isolation."""
+    plugin = _HeadExtrasPlugin()
+    plugin_manager.register(plugin, name="test_head_extras")
+    yield plugin
+    plugin_manager.unregister(name="test_head_extras")
+
+
+def test_plugin_html_injected_into_head(client, live_server, measure_memory_usage, datastore_path, head_extras_plugin):
+    """get_html_head_extras output must appear inside <head> in the rendered page."""
+    res = client.get(url_for("watchlist.index"), follow_redirects=True)
+    assert res.status_code == 200
+    assert b'id="test-head-extra-css"' in res.data, "Plugin <link> tag missing from rendered page"
+    assert b'id="test-head-extra-js"' in res.data,  "Plugin <script> tag missing from rendered page"
+
+    head_end = res.data.find(b'</head>')
+    assert head_end != -1
+    for marker in (b'id="test-head-extra-css"', b'id="test-head-extra-js"'):
+        pos = res.data.find(marker)
+        assert pos != -1 and pos < head_end, f"{marker} must appear before </head>"
+
+
+def test_plugin_js_route_returns_correct_content(client, live_server, measure_memory_usage, datastore_path, plugin_routes):
+    """The plugin-registered JS route must return JS with the right Content-Type."""
+    res = client.get(url_for('test_plugin_my_module_content_js'))
+    assert res.status_code == 200
+    assert 'javascript' in res.content_type
+    assert _MY_JS.encode() in res.data
+
+
+def test_plugin_css_route_returns_correct_content(client, live_server, measure_memory_usage, datastore_path, plugin_routes):
+    """The plugin-registered CSS route must return CSS with the right Content-Type."""
+    res = client.get(url_for('test_plugin_my_module_content_css'))
+    assert res.status_code == 200
+    assert 'css' in res.content_type
+    assert _MY_CSS.encode() in res.data
+
+
+def test_no_extras_without_plugin(client, live_server, measure_memory_usage, datastore_path):
+    """With no hookimpl registered the markers must not appear (isolation check)."""
+    res = client.get(url_for("watchlist.index"), follow_redirects=True)
+    assert b'id="test-head-extra-css"' not in res.data
+    assert b'id="test-head-extra-js"' not in res.data

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -11,10 +11,10 @@ from changedetectionio.tests.util import set_original_response, set_modified_res
     set_longer_modified_response, delete_all_watches
 
 import logging
-
+import os
 
 # NOTE - RELIES ON mailserver as hostname running, see github build recipes
-smtp_test_server = 'mailserver'
+smtp_test_server = os.getenv('SMTP_TEST_MAILSERVER', 'mailserver')
 
 ALL_MARKUP_TOKENS = ''.join(f"TOKEN: '{t}'\n{{{{{t}}}}}\n" for t in NotificationContextData().keys())
 

changedetectionio/tests/test_backend.py

@@ -422,3 +422,28 @@ def test_plaintext_even_if_xml_content_and_can_apply_filters(client, live_server
     assert b'<foobar' not in res.data
 
     res = delete_all_watches(client)
+
+
+def test_last_error_cleared_on_same_checksum(client, live_server, datastore_path):
+    """last_error should be cleared even when content is unchanged (checksumFromPreviousCheckWasTheSame path)"""
+    set_original_response(datastore_path=datastore_path)
+
+    uuid = client.application.config.get('DATASTORE').add_watch(url=url_for('test_endpoint', _external=True))
+
+    # First check - establishes baseline checksum
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    # Inject a stale last_error directly (simulates a prior failed check)
+    datastore = client.application.config.get('DATASTORE')
+    datastore.update_watch(uuid=uuid, update_obj={'last_error': 'Some previous error'})
+    assert datastore.data['watching'][uuid].get('last_error') == 'Some previous error'
+
+    # Second check - same content, so checksumFromPreviousCheckWasTheSame will fire
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    # last_error must be cleared even though no change was detected
+    assert datastore.data['watching'][uuid].get('last_error') == False
+
+    delete_all_watches(client)

changedetectionio/tests/test_import.py

@@ -214,3 +214,85 @@ def test_import_watchete_xlsx(client, live_server, measure_memory_usage, datasto
             assert watch.get('fetch_backend') == 'system' # uses default if blank
 
     delete_all_watches(client)
+
+
+def test_import_wachete_xlsx_row_counter(client, live_server, measure_memory_usage, datastore_path):
+    """Row counter in Wachete XLSX import must advance even after a failed row.
+
+    Regression: row_id was only incremented in the try/else (on success), so
+    after any failure the counter froze and all subsequent errors cited the
+    stale number.  With the enumerate() fix, row 5 must say "row 5", not "row 3".
+    """
+    import openpyxl
+
+    wb = openpyxl.Workbook()
+    ws = wb.active
+    # Header row (row 1)
+    ws.append(['Name', 'Id', 'Url', 'Interval (min)', 'XPath', 'Dynamic Wachet', 'Portal Wachet', 'Folder'])
+    # Row 2: valid
+    ws.append(['Site A', '001', 'https://example.com/a', 60, None, None, None, None])
+    # Row 3: bad URL — must report row 3
+    ws.append(['Site B', '002', 'not-a-valid-url', 60, None, None, None, None])
+    # Row 4: valid
+    ws.append(['Site C', '003', 'https://example.com/c', 60, None, None, None, None])
+    # Row 5: bad URL — must report row 5, not "row 3" (the pre-fix stale value)
+    ws.append(['Site D', '004', 'also-not-valid', 60, None, None, None, None])
+
+    xlsx_bytes = io.BytesIO()
+    wb.save(xlsx_bytes)
+    xlsx_bytes.seek(0)
+
+    res = client.post(
+        url_for("imports.import_page"),
+        data={'file_mapping': 'wachete', 'xlsx_file': (xlsx_bytes, 'test.xlsx')},
+        follow_redirects=True,
+    )
+
+    assert b'2 imported from Wachete .xlsx' in res.data
+    assert b'Error processing row number 3' in res.data
+    assert b'Error processing row number 5' in res.data
+
+    delete_all_watches(client)
+
+
+def test_import_custom_xlsx_row_counter(client, live_server, measure_memory_usage, datastore_path):
+    """Row counter in custom XLSX import must reflect the actual row, not always row 1.
+
+    Regression: row_i was incremented in the else clause of the *outer* try/except
+    (which only fired once, after the whole loop), so every URL-validation error
+    inside the loop reported "row 1".  With enumerate() the third row must say
+    "row 3", not "row 1".
+    """
+    import openpyxl
+
+    wb = openpyxl.Workbook()
+    ws = wb.active
+    # Row 1: bad URL — must report row 1
+    ws.append(['not-valid-url-row1'])
+    # Row 2: valid
+    ws.append(['https://example.com/b'])
+    # Row 3: bad URL — must report row 3, not "row 1" (the pre-fix value)
+    ws.append(['not-valid-url-row3'])
+    # Row 4: valid
+    ws.append(['https://example.com/d'])
+
+    xlsx_bytes = io.BytesIO()
+    wb.save(xlsx_bytes)
+    xlsx_bytes.seek(0)
+
+    res = client.post(
+        url_for("imports.import_page"),
+        data={
+            'file_mapping': 'custom',
+            'custom_xlsx[col_0]': '1',
+            'custom_xlsx[col_type_0]': 'url',
+            'xlsx_file': (xlsx_bytes, 'test.xlsx'),
+        },
+        follow_redirects=True,
+    )
+
+    assert b'2 imported from custom .xlsx' in res.data
+    assert b'Error processing row number 1' in res.data
+    assert b'Error processing row number 3' in res.data
+
+    delete_all_watches(client)

changedetectionio/tests/test_security.py

@@ -50,6 +50,85 @@ def test_favicon(client, live_server, measure_memory_usage, datastore_path):
     res = client.get(url_for('static_content', group='js', filename='../styles/styles.css'))
     assert res.status_code != 200
 
+def test_favicon_inline_data_uri(client, live_server, measure_memory_usage, datastore_path):
+    """
+    bump_favicon() must handle a data URI as the url parameter.
+    Previously this logged "Cant work out file extension from 'data:image/png;base64,...'" and bailed.
+    The mime_type from the data URI should be used to pick the correct extension.
+    """
+    import base64
+    import os
+
+    # 1x1 transparent PNG (minimal valid PNG bytes)
+    PNG_BYTES = (
+        b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01'
+        b'\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\nIDATx\x9cc\x00\x01'
+        b'\x00\x00\x05\x00\x01\r\n-\xb4\x00\x00\x00\x00IEND\xaeB`\x82'
+    )
+    png_b64 = base64.b64encode(PNG_BYTES).decode()
+    data_uri = f"data:image/png;base64,{png_b64}"
+
+    uuid = client.application.config.get('DATASTORE').add_watch(url='https://localhost')
+    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
+
+    # Should NOT raise / bail — must save as favicon.png
+    watch.bump_favicon(url=data_uri, favicon_base_64=png_b64, mime_type='image/png')
+
+    favicon_fname = watch.get_favicon_filename()
+    assert favicon_fname is not None, "Favicon should have been saved"
+    assert favicon_fname.endswith('.png'), f"Expected .png extension, got: {favicon_fname}"
+
+    full_path = os.path.join(watch.data_dir, favicon_fname)
+    assert os.path.getsize(full_path) == len(PNG_BYTES)
+
+    # Also verify it's served correctly via the static route
+    res = client.get(url_for('static_content', group='favicon', filename=uuid))
+    assert res.status_code == 200
+    assert res.data == PNG_BYTES
+
+
+def test_favicon_mime_type_overrides_url_extension(client, live_server, measure_memory_usage, datastore_path):
+    """
+    mime_type parameter takes precedence over the URL path extension.
+    A URL ending in .ico but with mime_type='image/png' should save as .png.
+    """
+    import base64
+    import os
+
+    PNG_BYTES = (
+        b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01'
+        b'\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\nIDATx\x9cc\x00\x01'
+        b'\x00\x00\x05\x00\x01\r\n-\xb4\x00\x00\x00\x00IEND\xaeB`\x82'
+    )
+    png_b64 = base64.b64encode(PNG_BYTES).decode()
+
+    uuid = client.application.config.get('DATASTORE').add_watch(url='https://localhost')
+    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
+
+    watch.bump_favicon(url='https://example.com/favicon.ico', favicon_base_64=png_b64, mime_type='image/png')
+
+    favicon_fname = watch.get_favicon_filename()
+    assert favicon_fname is not None
+    assert favicon_fname.endswith('.png'), f"mime_type should override URL extension, got: {favicon_fname}"
+
+
+def test_favicon_oversized_rejected(client, live_server, measure_memory_usage, datastore_path):
+    """Favicons larger than 1 MB must be silently dropped."""
+    import base64
+    import os
+
+    oversized = b'\x00' * (1 * 1024 * 1024 + 1)
+    oversized_b64 = base64.b64encode(oversized).decode()
+
+    uuid = client.application.config.get('DATASTORE').add_watch(url='https://localhost')
+    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
+
+    result = watch.bump_favicon(url='https://example.com/big.png', favicon_base_64=oversized_b64, mime_type='image/png')
+
+    assert result is None, "bump_favicon should return None for oversized favicon"
+    assert watch.get_favicon_filename() is None, "No favicon file should have been written"
+
+
 def test_bad_access(client, live_server, measure_memory_usage, datastore_path):
 
     res = client.post(

changedetectionio/tests/test_tag_url_match.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""
+Integration tests for auto-applying tags to watches by URL pattern matching.
+
+Verifies:
+ - A tag with url_match_pattern shows on the watch overview list (via get_all_tags_for_watch)
+ - The auto-applied tag appears on the watch edit page
+ - A watch whose URL does NOT match the pattern does not get the tag
+"""
+
+import json
+from flask import url_for
+from .util import set_original_response, live_server_setup
+
+
+def test_tag_url_pattern_shows_in_overview(client, live_server, measure_memory_usage, datastore_path):
+    """Tag with a matching url_match_pattern must appear in the watch overview row."""
+    set_original_response(datastore_path=datastore_path)
+
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+
+    # Create a tag with a URL match pattern
+    res = client.post(
+        url_for("tag"),
+        data=json.dumps({"title": "Auto GitHub", "url_match_pattern": "*github.com*"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
+    )
+    assert res.status_code == 201, res.data
+    tag_uuid = res.json['uuid']
+
+    # Add a watch that matches the pattern
+    res = client.post(
+        url_for("createwatch"),
+        data=json.dumps({"url": "https://github.com/someuser/repo"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
+    )
+    assert res.status_code == 201, res.data
+    matching_watch_uuid = res.json['uuid']
+
+    # Add a watch that does NOT match
+    res = client.post(
+        url_for("createwatch"),
+        data=json.dumps({"url": "https://example.com/page"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
+    )
+    assert res.status_code == 201, res.data
+    non_matching_watch_uuid = res.json['uuid']
+
+    # Watch overview — the tag label must appear in the matching watch's row
+    res = client.get(url_for("watchlist.index"))
+    assert res.status_code == 200
+    html = res.get_data(as_text=True)
+
+    # The tag title should appear somewhere on the page (it's rendered per-watch via get_all_tags_for_watch)
+    assert "Auto GitHub" in html, "Auto-matched tag title must appear in watch overview"
+
+    # Verify via the datastore directly that get_all_tags_for_watch returns the pattern-matched tag
+    datastore = live_server.app.config['DATASTORE']
+
+    matching_tags = datastore.get_all_tags_for_watch(matching_watch_uuid)
+    assert tag_uuid in matching_tags, "Pattern-matched tag must be returned for matching watch"
+
+    non_matching_tags = datastore.get_all_tags_for_watch(non_matching_watch_uuid)
+    assert tag_uuid not in non_matching_tags, "Pattern-matched tag must NOT appear for non-matching watch"
+
+
+def test_auto_applied_tag_shows_on_watch_edit(client, live_server, measure_memory_usage, datastore_path):
+    """The watch edit page must show auto-applied tags (from URL pattern) separately."""
+    set_original_response(datastore_path=datastore_path)
+
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+
+    res = client.post(
+        url_for("tag"),
+        data=json.dumps({"title": "Auto Docs", "url_match_pattern": "*docs.example.com*"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
+    )
+    assert res.status_code == 201, res.data
+
+    res = client.post(
+        url_for("createwatch"),
+        data=json.dumps({"url": "https://docs.example.com/guide"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
+    )
+    assert res.status_code == 201, res.data
+    watch_uuid = res.json['uuid']
+
+    # Watch edit page must mention the auto-applied tag
+    res = client.get(url_for("ui.ui_edit.edit_page", uuid=watch_uuid))
+    assert res.status_code == 200
+    html = res.get_data(as_text=True)
+
+    assert "Auto Docs" in html, "Auto-applied tag name must appear on watch edit page"
+    assert "automatically applied" in html.lower() or "auto" in html.lower(), \
+        "Watch edit page must indicate the tag is auto-applied by pattern"
+
+
+def test_multiple_pattern_tags_all_applied(client, live_server, measure_memory_usage, datastore_path):
+    """A watch matching multiple tag patterns must receive all of them, not just the first."""
+    set_original_response(datastore_path=datastore_path)
+
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+
+    # Two tags with different patterns that both match the same URL
+    res = client.post(
+        url_for("tag"),
+        data=json.dumps({"title": "Org Docs", "url_match_pattern": "*docs.*"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
+    )
+    assert res.status_code == 201, res.data
+    tag_docs_uuid = res.json['uuid']
+
+    res = client.post(
+        url_for("tag"),
+        data=json.dumps({"title": "Org Python", "url_match_pattern": "*python*"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
+    )
+    assert res.status_code == 201, res.data
+    tag_python_uuid = res.json['uuid']
+
+    # A third tag whose pattern does NOT match
+    res = client.post(
+        url_for("tag"),
+        data=json.dumps({"title": "Org Rust", "url_match_pattern": "*rust-lang*"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
+    )
+    assert res.status_code == 201, res.data
+    tag_rust_uuid = res.json['uuid']
+
+    # Watch URL matches both "docs" and "python" patterns but not "rust"
+    res = client.post(
+        url_for("createwatch"),
+        data=json.dumps({"url": "https://docs.python.org/3/library/fnmatch.html"}),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
+    )
+    assert res.status_code == 201, res.data
+    watch_uuid = res.json['uuid']
+
+    datastore = live_server.app.config['DATASTORE']
+    resolved = datastore.get_all_tags_for_watch(watch_uuid)
+
+    assert tag_docs_uuid in resolved, "First matching tag must be included"
+    assert tag_python_uuid in resolved, "Second matching tag must be included"
+    assert tag_rust_uuid not in resolved, "Non-matching tag must NOT be included"

changedetectionio/tests/test_xpath_selector.py

@@ -610,6 +610,11 @@ def test_xpath_blocked_functions_unit():
         "unparsed-text-available('file:///etc/passwd')",
         "doc('file:///etc/passwd')",
         "doc-available('file:///etc/passwd')",
+        "json-doc('file:///datastore/changedetection.json')",
+        "collection('file:///datastore/')",
+        "uri-collection('file:///datastore/')",
+        "transform(map{})",
+        "load-xquery-module('foo')",
         "environment-variable('PATH')",
         "available-environment-variables()",
     ]

changedetectionio/tests/unit/test_auth_decorator_order.py

@@ -0,0 +1,85 @@
+"""
+Static analysis test: verify @login_optionally_required is always applied
+AFTER (inner to) @blueprint.route(), not before it.
+
+In Flask, @route() must be the outermost decorator because it registers
+whatever function it receives. If @login_optionally_required is placed
+above @route(), the raw unprotected function gets registered and auth is
+silently bypassed (GHSA-jmrh-xmgh-x9j4).
+
+Correct order (route outermost, auth inner):
+    @blueprint.route('/path')
+    @login_optionally_required
+    def view(): ...
+
+Wrong order (auth never called):
+    @login_optionally_required   ← registered by route, then discarded
+    @blueprint.route('/path')
+    def view(): ...
+"""
+
+import ast
+import pathlib
+import pytest
+
+REPO_ROOT = pathlib.Path(__file__).parents[3]  # …/changedetection.io/
+SOURCE_ROOT = REPO_ROOT / "changedetectionio"
+
+
+def _is_route_decorator(node: ast.expr) -> bool:
+    """Return True if the decorator looks like @something.route(...)."""
+    return (
+        isinstance(node, ast.Call)
+        and isinstance(node.func, ast.Attribute)
+        and node.func.attr == "route"
+    )
+
+
+def _is_auth_decorator(node: ast.expr) -> bool:
+    """Return True if the decorator is @login_optionally_required."""
+    return isinstance(node, ast.Name) and node.id == "login_optionally_required"
+
+
+def collect_violations() -> list[str]:
+    violations = []
+
+    for path in SOURCE_ROOT.rglob("*.py"):
+        try:
+            tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
+        except SyntaxError:
+            continue
+
+        for node in ast.walk(tree):
+            if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                continue
+
+            decorators = node.decorator_list
+            auth_indices = [i for i, d in enumerate(decorators) if _is_auth_decorator(d)]
+            route_indices = [i for i, d in enumerate(decorators) if _is_route_decorator(d)]
+
+            # Bad order: auth decorator appears at a lower index (higher up) than a route decorator
+            for auth_idx in auth_indices:
+                for route_idx in route_indices:
+                    if auth_idx < route_idx:
+                        rel = path.relative_to(REPO_ROOT)
+                        violations.append(
+                            f"{rel}:{node.lineno} — `{node.name}`: "
+                            f"@login_optionally_required (line {decorators[auth_idx].lineno}) "
+                            f"is above @route (line {decorators[route_idx].lineno}); "
+                            f"auth wrapper will never be called"
+                        )
+
+    return violations
+
+
+def test_auth_decorator_order():
+    violations = collect_violations()
+    if violations:
+        msg = (
+            "\n\nFound routes where @login_optionally_required is placed ABOVE @blueprint.route().\n"
+            "This silently disables authentication — @route() registers the raw function\n"
+            "and the auth wrapper is never called.\n\n"
+            "Fix: move @blueprint.route() to be the outermost (topmost) decorator.\n\n"
+            + "\n".join(f"  • {v}" for v in violations)
+        )
+        pytest.fail(msg)

changedetectionio/tests/unit/test_conditions.py

@@ -64,7 +64,7 @@ class TestTriggerConditions(unittest.TestCase):
                 "conditions": [
                     {"operator": ">=", "field": "extracted_number", "value": "10"},
                     {"operator": "<=", "field": "extracted_number", "value": "5000"},
-                    {"operator": "in", "field": "page_text", "value": "rock"},
+                    {"operator": "in", "field": "page_filtered_text", "value": "rock"},
                     #{"operator": "starts_with", "field": "page_text", "value": "I saw"},
                 ]
             }

changedetectionio/tests/unit/test_notification_diff.py

@@ -15,7 +15,9 @@ from changedetectionio.diff import (
     CHANGED_PLACEMARKER_OPEN,
     CHANGED_PLACEMARKER_CLOSED,
     CHANGED_INTO_PLACEMARKER_OPEN,
-    CHANGED_INTO_PLACEMARKER_CLOSED
+    CHANGED_INTO_PLACEMARKER_CLOSED,
+    extract_changed_from,
+    extract_changed_to,
 )
 
 
@@ -381,5 +383,140 @@ Line 3 with tabs and spaces"""
         self.assertNotIn('[-Line 2-]', output)
         self.assertNotIn('[+Line 2+]', output)
 
+    def test_diff_changed_from_to_word_level(self):
+        """Primary use case: extract just the old/new value from a changed line (e.g. price monitoring)"""
+        before = "Widget costs $99.99 per month"
+        after  = "Widget costs $109.99 per month"
+
+        raw = diff.render_diff(before, after, word_diff=True)
+
+        self.assertEqual(extract_changed_from(raw), "$99.99")
+        self.assertEqual(extract_changed_to(raw),   "$109.99")
+
+    def test_diff_changed_from_to_multiple_changes(self):
+        """Multiple changed fragments on different lines are joined with newline.
+        An unchanged line between the two changes ensures each is a 1-to-1 replace,
+        so word_diff fires per line rather than falling back to multi-line block mode."""
+        before = "Price $99\nunchanged\nTax $5"
+        after  = "Price $149\nunchanged\nTax $12"
+
+        raw = diff.render_diff(before, after, word_diff=True)
+
+        self.assertEqual(extract_changed_from(raw), "$99\n$5")
+        self.assertEqual(extract_changed_to(raw),   "$149\n$12")
+
+    def test_diff_changed_from_to_pure_insert_delete(self):
+        """Pure line additions/deletions (no inline word diff) are also captured"""
+        before = "old line"
+        after  = "new line"
+
+        # word_diff=False forces line-level CHANGED markers
+        raw = diff.render_diff(before, after, word_diff=False)
+
+        self.assertEqual(extract_changed_from(raw), "old line")
+        self.assertEqual(extract_changed_to(raw),   "new line")
+
+    def test_diff_changed_from_to_similar_numbers(self):
+        """$90.00 → $9.00 must not produce a partial match like '0.00'.
+        The tokenizer splits on whitespace only, so '$90.00' and '$9.00' are
+        each a single atomic token — diff never sees their internal characters."""
+        before = "for sale $90.00"
+        after  = "for sale $9.00"
+
+        raw = diff.render_diff(before, after, word_diff=True)
+
+        self.assertEqual(extract_changed_from(raw), "$90.00")
+        self.assertEqual(extract_changed_to(raw),   "$9.00")
+
+    def test_diff_changed_from_to_whole_line_replaced(self):
+        """When every token on the line changed (no common tokens), render_inline_word_diff
+        takes the whole_line_replaced path using CHANGED/CHANGED_INTO markers instead of
+        REMOVED/ADDED. Extraction must still work via the alternation in the regex."""
+        before = "$99"
+        after  = "$109"
+
+        raw = diff.render_diff(before, after, word_diff=True)
+
+        self.assertEqual(extract_changed_from(raw), "$99")
+        self.assertEqual(extract_changed_to(raw),   "$109")
+
+    def test_diff_changed_from_to_multiple_words_same_line(self):
+        """When multiple words change on the same line all fragments are joined with newline.
+        'quick brown fox jumps' -> 'slow brown fox hops' gives 'quick\njumps' / 'slow\nhops'.
+        These tokens work best when a single value changes per line."""
+        before = "quick brown fox jumps"
+        after  = "slow brown fox hops"
+
+        raw = diff.render_diff(before, after, word_diff=True)
+
+        self.assertEqual(extract_changed_from(raw), "quick\njumps")
+        self.assertEqual(extract_changed_to(raw),   "slow\nhops")
+
+    def test_diff_changed_from_to_no_change(self):
+        """No changes → empty string"""
+        content = "nothing changed here"
+
+        raw = diff.render_diff(content, content, word_diff=True)
+
+        self.assertEqual(extract_changed_from(raw), "")
+        self.assertEqual(extract_changed_to(raw),   "")
+
+
+    def test_word_diff_no_prefix_whole_line_replaced(self):
+        """When include_change_type_prefix=False, word-level diffs for whole-line
+        replacements must not include placemarkers (issue #3816)."""
+        before = "73"
+        after = "100"
+
+        raw = diff.render_diff(before, after, word_diff=True, include_change_type_prefix=False)
+
+        self.assertNotIn('PLACEMARKER', raw)
+        # Should contain just the raw values separated by newline
+        self.assertIn('73', raw)
+        self.assertIn('100', raw)
+
+    def test_word_diff_no_prefix_inline_changes(self):
+        """When include_change_type_prefix=False, inline word-level diffs
+        must not include placemarkers (issue #3816)."""
+        before = "the price is 50 dollars"
+        after = "the price is 75 dollars"
+
+        raw = diff.render_diff(before, after, word_diff=True, include_change_type_prefix=False)
+
+        self.assertNotIn('PLACEMARKER', raw)
+        self.assertIn('50', raw)
+        self.assertIn('75', raw)
+
+    def test_word_diff_with_prefix_still_wraps(self):
+        """Default include_change_type_prefix=True must still wrap tokens."""
+        before = "73"
+        after = "100"
+
+        raw = diff.render_diff(before, after, word_diff=True, include_change_type_prefix=True)
+
+        self.assertIn('PLACEMARKER', raw)
+
+    def test_word_diff_no_prefix_exact_output(self):
+        """Pin exact output for include_change_type_prefix=False to catch regressions.
+
+        Whole-line replacement: old and new values separated by newline, no markers.
+        Inline partial replacement: equal tokens kept, changed tokens (both old and new)
+        appended without markers — this means old+new are concatenated in place.
+        """
+        # Whole-line replaced: both values on separate lines, clean
+        raw = diff.render_diff('73', '100', word_diff=True, include_change_type_prefix=False)
+        self.assertEqual(raw, '73\n100')
+
+        # Inline word replacement: equal context preserved, old+new token concatenated in-place
+        raw = diff.render_diff('the price is 50 dollars', 'the price is 75 dollars',
+                               word_diff=True, include_change_type_prefix=False)
+        self.assertEqual(raw, 'the price is 5075 dollars')
+
+        # Sanity: with prefix the whole-line case is fully wrapped
+        raw = diff.render_diff('73', '100', word_diff=True, include_change_type_prefix=True)
+        self.assertEqual(raw, '@changed_PLACEMARKER_OPEN73@changed_PLACEMARKER_CLOSED\n'
+                               '@changed_into_PLACEMARKER_OPEN100@changed_into_PLACEMARKER_CLOSED')
+
+
 if __name__ == '__main__':
     unittest.main()

changedetectionio/tests/unit/test_step_failure_notification.py

@@ -0,0 +1,68 @@
+"""
+Unit test for send_step_failure_notification regression.
+
+Before the fix, line 499 called self._check_cascading_vars('notification_format', watch)
+which raises AttributeError because _check_cascading_vars is a module-level function,
+not a method of NotificationService.
+"""
+
+import queue
+from unittest.mock import MagicMock
+
+
+def _make_datastore(watch_uuid, notification_url):
+    """Minimal datastore mock that NotificationService and _check_cascading_vars need."""
+    watch = MagicMock()
+    watch.get = lambda key, default=None: {
+        'uuid': watch_uuid,
+        'url': 'https://example.com',
+        'notification_urls': [notification_url],
+        'notification_format': '',
+        'notification_muted': False,
+    }.get(key, default)
+    watch.__getitem__ = lambda self, key: watch.get(key)
+
+    datastore = MagicMock()
+    datastore.data = {
+        'watching': {watch_uuid: watch},
+        'settings': {
+            'application': {
+                'notification_urls': [],
+                'notification_format': 'text',
+                'filter_failure_notification_threshold_attempts': 3,
+            }
+        }
+    }
+    datastore.get_all_tags_for_watch.return_value = {}
+    return datastore, watch
+
+
+def test_send_step_failure_notification_does_not_raise():
+    """send_step_failure_notification must not raise AttributeError (wrong self. prefix on module-level function)."""
+    from changedetectionio.notification_service import NotificationService
+
+    watch_uuid = 'test-uuid-1234'
+    notification_q = queue.Queue()
+    datastore, _ = _make_datastore(watch_uuid, 'post://localhost/test')
+    service = NotificationService(datastore=datastore, notification_q=notification_q)
+
+    # Before the fix this raised:
+    # AttributeError: 'NotificationService' object has no attribute '_check_cascading_vars'
+    service.send_step_failure_notification(watch_uuid=watch_uuid, step_n=0)
+
+
+def test_send_step_failure_notification_queues_item():
+    """A notification object should be placed on the queue when URLs are configured."""
+    from changedetectionio.notification_service import NotificationService
+
+    watch_uuid = 'test-uuid-5678'
+    notification_q = queue.Queue()
+    datastore, _ = _make_datastore(watch_uuid, 'post://localhost/test')
+    service = NotificationService(datastore=datastore, notification_q=notification_q)
+
+    service.send_step_failure_notification(watch_uuid=watch_uuid, step_n=1)
+
+    assert not notification_q.empty(), "Expected a notification to be queued"
+    item = notification_q.get_nowait()
+    assert 'notification_title' in item
+    assert 'position 2' in item['notification_title']

changedetectionio/tests/unit/test_tag_url_match.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+# run from dir above changedetectionio/ dir
+# python3 -m unittest changedetectionio.tests.unit.test_tag_url_match
+
+import unittest
+from changedetectionio.model.Tag import model as TagModel
+
+
+def make_tag(pattern):
+    """Minimal Tag instance for testing matches_url — skips datastore wiring."""
+    tag = TagModel.__new__(TagModel)
+    dict.__init__(tag)
+    tag['url_match_pattern'] = pattern
+    return tag
+
+
+class TestTagUrlMatch(unittest.TestCase):
+
+    def test_wildcard_matches(self):
+        tag = make_tag('*example.com*')
+        self.assertTrue(tag.matches_url('https://example.com/page'))
+        self.assertTrue(tag.matches_url('https://www.example.com/shop/item'))
+        self.assertFalse(tag.matches_url('https://other.com/page'))
+
+    def test_wildcard_case_insensitive(self):
+        tag = make_tag('*EXAMPLE.COM*')
+        self.assertTrue(tag.matches_url('https://example.com/page'))
+
+    def test_substring_match(self):
+        tag = make_tag('github.com/myorg')
+        self.assertTrue(tag.matches_url('https://github.com/myorg/repo'))
+        self.assertFalse(tag.matches_url('https://github.com/otherorg/repo'))
+
+    def test_substring_case_insensitive(self):
+        tag = make_tag('GitHub.com/MyOrg')
+        self.assertTrue(tag.matches_url('https://github.com/myorg/repo'))
+
+    def test_empty_pattern_never_matches(self):
+        tag = make_tag('')
+        self.assertFalse(tag.matches_url('https://example.com'))
+
+    def test_empty_url_never_matches(self):
+        tag = make_tag('*example.com*')
+        self.assertFalse(tag.matches_url(''))
+
+    def test_question_mark_wildcard(self):
+        tag = make_tag('https://example.com/item-?')
+        self.assertTrue(tag.matches_url('https://example.com/item-1'))
+        self.assertFalse(tag.matches_url('https://example.com/item-12'))
+
+    def test_substring_is_broad(self):
+        """Plain substring matching is intentionally broad — 'evil.com' matches anywhere
+        in the URL string, including 'notevil.com'. Users who need precise domain matching
+        should use a wildcard pattern like '*://evil.com/*' instead."""
+        tag = make_tag('evil.com')
+        self.assertTrue(tag.matches_url('https://evil.com/page'))
+        self.assertTrue(tag.matches_url('https://notevil.com'))  # substring match — expected
+
+    def test_precise_domain_match_with_wildcard(self):
+        """Use wildcard pattern for precise domain matching to avoid substring surprises."""
+        tag = make_tag('*://evil.com/*')
+        self.assertTrue(tag.matches_url('https://evil.com/page'))
+        self.assertFalse(tag.matches_url('https://notevil.com/page'))
+
+
+if __name__ == '__main__':
+    unittest.main()

changedetectionio/translations/README.md

@@ -76,7 +76,9 @@ These commands read settings from `../../setup.cfg` automatically.
 - `en_US` - English (US)
 - `fr` - French (Français)
 - `it` - Italian (Italiano)
+- `ja` - Japanese (日本語)
 - `ko` - Korean (한국어)
+- `pt_BR` - Portuguese (Brasil)
 - `zh` - Chinese Simplified (中文简体)
 - `zh_Hant_TW` - Chinese Traditional (繁體中文)
 

changedetectionio/translations/de/LC_MESSAGES/messages.po

@@ -1617,7 +1617,7 @@ msgstr "Bereich zeichnen"
 
 #: changedetectionio/blueprint/ui/templates/edit.html
 msgid "Clear selection"
-msgstr "Klare Auswahl"
+msgstr "Auswahl löschen"
 
 #: changedetectionio/blueprint/ui/templates/edit.html
 msgid "One moment, fetching screenshot and element information.."

changedetectionio/worker.py

@@ -284,6 +284,9 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                     logger.debug(f'[{uuid}] - checksumFromPreviousCheckWasTheSame - Checksum from previous check was the same, nothing todo here.')
                     # Reset the edited flag since we successfully completed the check
                     watch.reset_watch_edited_flag()
+                    # Page was fetched successfully - clear any previous error state
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': False})
+                    cleanup_error_artifacts(uuid, datastore)
                     
                 except content_fetchers_exceptions.BrowserConnectError as e:
                     datastore.update_watch(uuid=uuid,
@@ -481,7 +484,8 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                     # Store favicon if necessary
                     if update_handler.fetcher.favicon_blob and update_handler.fetcher.favicon_blob.get('base64'):
                         watch.bump_favicon(url=update_handler.fetcher.favicon_blob.get('url'),
-                                           favicon_base_64=update_handler.fetcher.favicon_blob.get('base64')
+                                           favicon_base_64=update_handler.fetcher.favicon_blob.get('base64'),
+                                           mime_type=update_handler.fetcher.favicon_blob.get('mime_type')
                                            )
 
                     datastore.update_watch(uuid=uuid, update_obj=final_updates)

docker-compose.yml

@@ -28,7 +28,7 @@ services:
   #      - PLAYWRIGHT_DRIVER_URL=ws://browser-sockpuppet-chrome:3000
   #
   #
-  #       Alternative WebDriver/selenium URL, do not use "'s or 's! (old, deprecated, does not support screenshots very well)
+  #       Alternative WebDriver/selenium URL, do not use "'s or 's! (old, deprecated, does not support screenshots very well, Can't handle custom headers etc)
   #      - WEBDRIVER_URL=http://browser-selenium-chrome:4444/wd/hub
   #
   #       WebDriver proxy settings webdriver_proxyType, webdriver_ftpProxy, webdriver_noProxy,

docs/api-spec.yaml

@@ -725,6 +725,13 @@ components:
                 - true: Tag settings override watch settings
                 - false: Tag settings do not override (watches use their own settings)
                 - null: Not decided yet / inherit default behavior
+            url_match_pattern:
+              type: string
+              description: |
+                Automatically apply this tag to any watch whose URL matches this pattern.
+                Supports fnmatch wildcards (* and ?): e.g. *://example.com/* or github.com/myorg.
+                Plain strings are matched as case-insensitive substrings.
+                Leave empty to disable auto-matching.
             # Future: Aggregated statistics from all watches with this tag
             # check_count:
             #   type: integer

requirements.txt

@@ -98,7 +98,7 @@ pytest-flask ~=1.3
 pytest-mock ~=3.15
 
 # OpenAPI validation support
-openapi-core[flask] ~= 0.22
+openapi-core[flask] ~= 0.23
 
 loguru