Update store.py

.github/test/Dockerfile-alpine

@@ -18,6 +18,7 @@ RUN \
     libxslt-dev \
     openssl-dev \
     python3-dev \
+    file \
     zip \
     zlib-dev && \
   apk add --update --no-cache \

.github/workflows/test-container-build.yml

@@ -23,8 +23,28 @@ on:
   # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
   # @todo: some kind of path filter for requirements.txt and Dockerfile
 jobs:
-  test-container-build:
+  builder:
+    name: Build ${{ matrix.platform }} (${{ matrix.dockerfile == './Dockerfile' && 'main' || 'alpine' }})
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          # Main Dockerfile platforms
+          - platform: linux/amd64
+            dockerfile: ./Dockerfile
+          - platform: linux/arm64
+            dockerfile: ./Dockerfile
+          - platform: linux/arm/v7
+            dockerfile: ./Dockerfile
+          - platform: linux/arm/v8
+            dockerfile: ./Dockerfile
+          - platform: linux/arm64/v8
+            dockerfile: ./Dockerfile
+          # Alpine Dockerfile platforms (musl via alpine check)
+          - platform: linux/amd64
+            dockerfile: ./.github/test/Dockerfile-alpine
+          - platform: linux/arm64
+            dockerfile: ./.github/test/Dockerfile-alpine
     steps:
         - uses: actions/checkout@v4
         - name: Set up Python 3.11
@@ -47,26 +67,14 @@ jobs:
             version: latest
             driver-opts: image=moby/buildkit:master
 
-        # https://github.com/dgtlmoon/changedetection.io/pull/1067
-        # Check we can still build under alpine/musl
-        - name: Test that the docker containers can build (musl via alpine check)
-          id: docker_build_musl
-          uses: docker/build-push-action@v6
-          with:
-            context: ./
-            file: ./.github/test/Dockerfile-alpine
-            platforms: linux/amd64,linux/arm64
-            cache-from: type=gha
-            cache-to: type=gha,mode=max
-
-        - name: Test that the docker containers can build
+        - name: Test that the docker containers can build (${{ matrix.platform }} - ${{ matrix.dockerfile }})
           id: docker_build
           uses: docker/build-push-action@v6
           # https://github.com/docker/build-push-action#customizing
           with:
             context: ./
-            file: ./Dockerfile
-            platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
+            file: ${{ matrix.dockerfile }}
+            platforms: ${{ matrix.platform }}
             cache-from: type=gha
             cache-to: type=gha,mode=max
 

.github/workflows/test-stack-reusable-workflow.yml

@@ -179,6 +179,26 @@ jobs:
 
           docker kill test-changedetectionio
 
+      - name: Test HTTPS SSL mode
+        run: |
+          openssl req -x509 -newkey rsa:4096 -keyout privkey.pem -out cert.pem -days 365 -nodes -subj "/CN=localhost"
+          docker run --name test-changedetectionio-ssl --rm -e SSL_CERT_FILE=cert.pem -e SSL_PRIVKEY_FILE=privkey.pem -p 5000:5000 -v ./cert.pem:/app/cert.pem -v ./privkey.pem:/app/privkey.pem -d test-changedetectionio
+          sleep 3
+          # Should return 0 (no error) when grep finds it
+          # -k because its self-signed
+          curl --retry-connrefused --retry 6 -k https://localhost:5000 -v|grep -q checkbox-uuid
+      
+          docker kill test-changedetectionio-ssl
+
+      - name: Test IPv6 Mode
+        run: |
+          # IPv6 - :: bind to all interfaces inside container (like 0.0.0.0), ::1 would be localhost only
+          docker run --name test-changedetectionio-ipv6 --rm -p 5000:5000 -e LISTEN_HOST=:: -d test-changedetectionio
+          sleep 3
+          # Should return 0 (no error) when grep finds it on localhost
+          curl --retry-connrefused --retry 6 http://[::1]:5000 -v|grep -q checkbox-uuid
+          docker kill test-changedetectionio-ipv6
+
       - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
         run: |
           

Dockerfile

@@ -16,6 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libssl-dev \
     libxslt-dev \
     make \
+    patch \
     zlib1g-dev
 
 RUN mkdir /install
@@ -53,6 +54,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     locales \
     # For pdftohtml
     poppler-utils \
+    # favicon type detection and other uses
+    file \
     zlib1g \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
 

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2025 Web Technologies s.r.o.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README-pip.md

@@ -1,11 +1,21 @@
-## Web Site Change Detection, Monitoring and Notification.
+# Monitor website changes
 
-Live your data-life pro-actively, track website content changes and receive notifications via Discord, Email, Slack, Telegram and 70+ more
+Detect WebPage Changes Automatically — Monitor Web Page Changes in Real Time
+
+Monitor websites for updates — get notified via Discord, Email, Slack, Telegram, Webhook and many more.
+
+Detect web page content changes and get instant alerts.
+
+
+[Changedetection.io is the best tool to monitor web-pages for changes](https://changedetection.io) Track website content changes and receive notifications via Discord, Email, Slack, Telegram and 90+ more
+
+Ideal for monitoring price changes, content edits, conditional changes and more.
 
 [<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring, list of websites with changes"  title="Self-hosted web page change monitoring, list of websites with changes"  />](https://changedetection.io)
 
 
-[**Don't have time? Let us host it for you! try our extremely affordable subscription use our proxies and support!**](https://changedetection.io) 
+[**Don't have time? Try our extremely affordable subscription use our proxies and support!**](https://changedetection.io) 
+
 
 
 ### Target specific parts of the webpage using the Visual Selector tool.

README.md

@@ -1,11 +1,13 @@
-## Web Site Change Detection, Restock monitoring and notifications.
+# Detect Website Changes Automatically — Monitor Web Page Changes in Real Time
 
-**_Detect website content changes and perform meaningful actions - trigger notifications via Discord, Email, Slack, Telegram, API calls and many more._**
+Monitor websites for updates — get notified via Discord, Email, Slack, Telegram, Webhook and many more.
 
-_Live your data-life pro-actively._ 
+**Detect web page content changes and get instant alerts.**  
+
+Ideal for monitoring price changes, content edits, conditional changes and more.
 
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web site page change monitoring"  title="Self-hosted web site page change monitoring"  />](https://changedetection.io?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Web site page change monitoring"  title="Web site page change monitoring"  />](https://changedetection.io?src=github)
 
 [![Release Version][release-shield]][release-link] [![Docker Pulls][docker-pulls]][docker-link] [![License][license-shield]](LICENSE.md)
 
@@ -13,6 +15,7 @@ _Live your data-life pro-actively._
 
 [**Get started with website page change monitoring straight away. Don't have time? Try our $8.99/month subscription, use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_
 
+
 - Chrome browser included.
 - Nothing to install, access via browser login after signup.
 - Super fast, no registration needed setup.
@@ -99,9 +102,7 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W
 - Configurable [proxy per watch](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration)
 - Send a screenshot with the notification when a change is detected in the web page
 
-We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.
-
-[Oxylabs](https://oxylabs.go2cloud.org/SH2d) is also an excellent proxy provider and well worth using, they offer Residental, ISP, Rotating and many other proxy types to suit your project. 
+We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $150 using our signup link.
 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.50.2'
+__version__ = '0.50.6'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
@@ -65,8 +65,7 @@ def main():
 
     datastore_path = None
     do_cleanup = False
-    host = "0.0.0.0"
-    ipv6_enabled = False
+    host = os.environ.get("LISTEN_HOST", "0.0.0.0").strip()
     port = int(os.environ.get('PORT', 5000))
     ssl_mode = False
 
@@ -108,10 +107,6 @@ def main():
         if opt == '-d':
             datastore_path = arg
 
-        if opt == '-6':
-            logger.success("Enabling IPv6 listen support")
-            ipv6_enabled = True
-
         # Cleanup (remove text files that arent in the index)
         if opt == '-c':
             do_cleanup = True
@@ -123,6 +118,20 @@ def main():
         if opt == '-l':
             logger_level = int(arg) if arg.isdigit() else arg.upper()
 
+
+    logger.success(f"changedetection.io version {get_version()} starting.")
+    # Launch using SocketIO run method for proper integration (if enabled)
+    ssl_cert_file = os.getenv("SSL_CERT_FILE", 'cert.pem')
+    ssl_privkey_file = os.getenv("SSL_PRIVKEY_FILE", 'privkey.pem')
+    if os.getenv("SSL_CERT_FILE") and os.getenv("SSL_PRIVKEY_FILE"):
+        ssl_mode = True
+
+    # SSL mode could have been set by -s too, therefor fallback to default values
+    if ssl_mode:
+        if not os.path.isfile(ssl_cert_file) or not os.path.isfile(ssl_privkey_file):
+            logger.critical(f"Cannot start SSL/HTTPS mode, Please be sure that {ssl_cert_file}' and '{ssl_privkey_file}' exist in in {os.getcwd()}")
+            os._exit(2)
+
     # Without this, a logger will be duplicated
     logger.remove()
     try:
@@ -222,19 +231,19 @@ def main():
 
 
     # SocketIO instance is already initialized in flask_app.py
-
-    # Launch using SocketIO run method for proper integration (if enabled)
     if socketio_server:
         if ssl_mode:
-            socketio.run(app, host=host, port=int(port), debug=False, 
-                        certfile='cert.pem', keyfile='privkey.pem', allow_unsafe_werkzeug=True)
+            logger.success(f"SSL mode enabled, attempting to start with '{ssl_cert_file}' and '{ssl_privkey_file}' in {os.getcwd()}")
+            socketio.run(app, host=host, port=int(port), debug=False,
+                         ssl_context=(ssl_cert_file, ssl_privkey_file), allow_unsafe_werkzeug=True)
         else:
             socketio.run(app, host=host, port=int(port), debug=False, allow_unsafe_werkzeug=True)
     else:
         # Run Flask app without Socket.IO if disabled
         logger.info("Starting Flask app without Socket.IO server")
         if ssl_mode:
-            app.run(host=host, port=int(port), debug=False, 
-                   ssl_context=('cert.pem', 'privkey.pem'))
+            logger.success(f"SSL mode enabled, attempting to start with '{ssl_cert_file}' and '{ssl_privkey_file}' in {os.getcwd()}")
+            app.run(host=host, port=int(port), debug=False,
+                    ssl_context=(ssl_cert_file, ssl_privkey_file))
         else:
             app.run(host=host, port=int(port), debug=False)

changedetectionio/api/Watch.py

@@ -5,7 +5,7 @@ from flask_expects_json import expects_json
 from changedetectionio import queuedWatchMetaData
 from changedetectionio import worker_handler
 from flask_restful import abort, Resource
-from flask import request, make_response
+from flask import request, make_response, send_from_directory
 import validators
 from . import auth
 import copy
@@ -191,6 +191,47 @@ class WatchSingleHistory(Resource):
 
         return response
 
+class WatchFavicon(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    def get(self, uuid):
+        """
+        @api {get} /api/v1/watch/<string:uuid>/favicon Get Favicon for a watch
+        @apiDescription Requires watch `uuid`
+        @apiExample {curl} Example usage:
+            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/favicon -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+        @apiName Get latest Favicon
+        @apiGroup Watch History
+        @apiSuccess (200) {String} OK
+        @apiSuccess (404) {String} ERR Not found
+        """
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message=f"No watch exists with the UUID of {uuid}")
+
+        favicon_filename = watch.get_favicon_filename()
+        if favicon_filename:
+            try:
+                import magic
+                mime = magic.from_file(
+                    os.path.join(watch.watch_data_dir, favicon_filename),
+                    mime=True
+                )
+            except ImportError:
+                # Fallback, no python-magic
+                import mimetypes
+                mime, encoding = mimetypes.guess_type(favicon_filename)
+
+            response = make_response(send_from_directory(watch.watch_data_dir, favicon_filename))
+            response.headers['Content-type'] = mime
+            response.headers['Cache-Control'] = 'max-age=300, must-revalidate'  # Cache for 5 minutes, then revalidate
+            return response
+
+        abort(404, message=f'No Favicon available for {uuid}')
+
 
 class CreateWatch(Resource):
     def __init__(self, **kwargs):

changedetectionio/api/__init__.py

@@ -26,7 +26,7 @@ schema_delete_notification_urls = copy.deepcopy(schema_notification_urls)
 schema_delete_notification_urls['required'] = ['notification_urls']
 
 # Import all API resources
-from .Watch import Watch, WatchHistory, WatchSingleHistory, CreateWatch
+from .Watch import Watch, WatchHistory, WatchSingleHistory, CreateWatch, WatchFavicon
 from .Tags import Tags, Tag
 from .Import import Import
 from .SystemInfo import SystemInfo

changedetectionio/async_update_worker.py

@@ -353,6 +353,12 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
                 except Exception as e:
                     pass
 
+                # Store favicon if necessary
+                if update_handler.fetcher.favicon_blob and update_handler.fetcher.favicon_blob.get('base64'):
+                    watch.bump_favicon(url=update_handler.fetcher.favicon_blob.get('url'),
+                                       favicon_base_64=update_handler.fetcher.favicon_blob.get('base64')
+                                       )
+
                 datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - fetch_start_time, 3),
                                                                'check_count': count})
 

changedetectionio/blueprint/browser_steps/__init__.py

@@ -228,10 +228,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                     watch.save_screenshot(screenshot=screenshot)
                     watch.save_xpath_data(data=xpath_data)
 
-        except playwright._impl._api_types.Error as e:
-            return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
         except Exception as e:
-            return make_response("Error fetching screenshot and element data - " + str(e), 401)
+            return make_response(f"Error fetching screenshot and element data - {str(e)}", 401)
 
         # SEND THIS BACK TO THE BROWSER
         output = {

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -1,8 +1,6 @@
 import os
 import time
 import re
-import sys
-import traceback
 from random import randint
 from loguru import logger
 
@@ -92,8 +90,32 @@ class steppable_browser_interface():
         if optional_value and ('{%' in optional_value or '{{' in optional_value):
             optional_value = jinja_render(template_str=optional_value)
 
+        # Trigger click and cautiously handle potential navigation
+        # This means the page redirects/reloads/changes JS etc etc
+        if call_action_name.startswith('click_'):
+            try:
+                # Set up navigation expectation before the click (like sync version)
+                async with self.page.expect_event("framenavigated", timeout=3000) as navigation_info:
+                    await action_handler(selector, optional_value)
+                
+                # Check if navigation actually occurred
+                try:
+                    await navigation_info.value  # This waits for the navigation promise
+                    logger.debug(f"Navigation occurred on {call_action_name}.")
+                except Exception:
+                    logger.debug(f"No navigation occurred within timeout when calling {call_action_name}, that's OK, continuing.")
+                    
+            except Exception as e:
+                # If expect_event itself times out, that means no navigation occurred - that's OK
+                if "framenavigated" in str(e) and "exceeded" in str(e):
+                    logger.debug(f"No navigation occurred within timeout when calling {call_action_name}, that's OK, continuing.")
+                else:
+                    raise e
+        else:
+            # Some other action that probably a navigation is not expected
+            await action_handler(selector, optional_value)
+
 
-        await action_handler(selector, optional_value)
         # Safely wait for timeout
         await self.page.wait_for_timeout(1.5 * 1000)
         logger.debug(f"Call action done in {time.time()-now:.2f}s")
@@ -110,7 +132,7 @@ class steppable_browser_interface():
 
     # Incase they request to go back to the start
     async def action_goto_site(self, selector=None, value=None):
-        return await self.action_goto_url(value=self.start_url)
+        return await self.action_goto_url(value=re.sub(r'^source:', '', self.start_url, flags=re.IGNORECASE))
 
     async def action_click_element_containing_text(self, selector=None, value=''):
         logger.debug("Clicking element containing text")
@@ -428,6 +450,9 @@ class browsersteps_live_ui(steppable_browser_interface):
         try:
             # Get screenshot first
             screenshot = await capture_full_page_async(page=self.page)
+            if not screenshot:
+                logger.error("No screenshot was retrieved :((")
+
             logger.debug(f"Time to get screenshot from browser {time.time() - now:.2f}s")
 
             # Then get interactive elements
@@ -450,6 +475,12 @@ class browsersteps_live_ui(steppable_browser_interface):
             
         except Exception as e:
             logger.error(f"Error getting current state: {str(e)}")
+            # If the page has navigated (common with logins) then the context is destroyed on navigation, continue
+            # I'm not sure that this is required anymore because we have the "expect navigation wrapper" at the top
+            if "Execution context was destroyed" in str(e):
+                logger.debug("Execution context was destroyed, most likely because of navigation, continuing...")
+            pass
+
             # Attempt recovery - force garbage collection
             try:
                 await self.page.request_gc()

changedetectionio/blueprint/settings/templates/settings.html

@@ -256,6 +256,11 @@ nav
                     {{ render_checkbox_field(form.application.form.ui.form.socket_io_enabled, class="socket_io_enabled") }}
                     <span class="pure-form-message-inline">Realtime UI Updates Enabled - (Restart required if this is changed)</span>
                 </div>
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.ui.form.favicons_enabled, class="") }}
+                    <span class="pure-form-message-inline">Enable or Disable Favicons next to the watch list</span>
+                </div>
+
             </div>
             <div class="tab-pane-inner" id="proxies">
                 <div id="recommended-proxy">

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -10,7 +10,7 @@
             <legend>Add a new organisational tag</legend>
             <div id="watch-add-wrapper-zone">
                 <div>
-                    {{ render_simple_field(form.name, placeholder="watch label / tag") }}
+                    {{ render_simple_field(form.name, placeholder="Watch group / tag") }}
                 </div>
                 <div>
                     {{ render_simple_field(form.save_button, title="Save" ) }}

changedetectionio/blueprint/ui/__init__.py

@@ -159,12 +159,20 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_handle
     def mark_all_viewed():
         # Save the current newest history as the most recently viewed
         with_errors = request.args.get('with_errors') == "1"
+        tag_limit = request.args.get('tag')
+        logger.debug(f"Limiting to tag {tag_limit}")
+        now = int(time.time())
         for watch_uuid, watch in datastore.data['watching'].items():
             if with_errors and not watch.get('last_error'):
                 continue
-            datastore.set_last_viewed(watch_uuid, int(time.time()))
 
-        return redirect(url_for('watchlist.index'))
+            if tag_limit and ( not watch.get('tags') or tag_limit not in watch['tags'] ):
+                logger.debug(f"Skipping watch {watch_uuid}")
+                continue
+
+            datastore.set_last_viewed(watch_uuid, now)
+
+        return redirect(url_for('watchlist.index', tag=tag_limit))
 
     @ui_blueprint.route("/delete", methods=['GET'])
     @login_optionally_required

changedetectionio/blueprint/ui/views.py

@@ -1,8 +1,7 @@
 from flask import Blueprint, request, redirect, url_for, flash, render_template, make_response, send_from_directory, abort
-from flask_login import current_user
 import os
 import time
-from copy import deepcopy
+from loguru import logger
 
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.auth_decorator import login_optionally_required
@@ -78,7 +77,42 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
         return output
 
-    @views_blueprint.route("/diff/<string:uuid>", methods=['GET', 'POST'])
+    @views_blueprint.route("/diff/<string:uuid>", methods=['POST'])
+    @login_optionally_required
+    def diff_history_page_build_report(uuid):
+        from changedetectionio import forms
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        try:
+            watch = datastore.data['watching'][uuid]
+        except KeyError:
+            flash("No history found for the specified link, bad link?", "error")
+            return redirect(url_for('watchlist.index'))
+
+        # For submission of requesting an extract
+        extract_form = forms.extractDataForm(request.form)
+        if not extract_form.validate():
+            flash("An error occurred, please see below.", "error")
+
+        else:
+            extract_regex = request.form.get('extract_regex').strip()
+            output = watch.extract_regex_from_all_history(extract_regex)
+            if output:
+                watch_dir = os.path.join(datastore.datastore_path, uuid)
+                response = make_response(send_from_directory(directory=watch_dir, path=output, as_attachment=True))
+                response.headers['Content-type'] = 'text/csv'
+                response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+                response.headers['Pragma'] = 'no-cache'
+                response.headers['Expires'] = "0"
+                return response
+
+            flash('Nothing matches that RegEx', 'error')
+        redirect(url_for('ui_views.diff_history_page', uuid=uuid) + '#extract')
+
+    @views_blueprint.route("/diff/<string:uuid>", methods=['GET'])
     @login_optionally_required
     def diff_history_page(uuid):
         from changedetectionio import forms
@@ -96,60 +130,31 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
         # For submission of requesting an extract
         extract_form = forms.extractDataForm(request.form)
-        if request.method == 'POST':
-            if not extract_form.validate():
-                flash("An error occurred, please see below.", "error")
-
-            else:
-                extract_regex = request.form.get('extract_regex').strip()
-                output = watch.extract_regex_from_all_history(extract_regex)
-                if output:
-                    watch_dir = os.path.join(datastore.datastore_path, uuid)
-                    response = make_response(send_from_directory(directory=watch_dir, path=output, as_attachment=True))
-                    response.headers['Content-type'] = 'text/csv'
-                    response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
-                    response.headers['Pragma'] = 'no-cache'
-                    response.headers['Expires'] = 0
-                    return response
-
-                flash('Nothing matches that RegEx', 'error')
-                redirect(url_for('ui_views.diff_history_page', uuid=uuid)+'#extract')
 
         history = watch.history
         dates = list(history.keys())
 
-        if len(dates) < 2:
-            flash("Not enough saved change detection snapshots to produce a report.", "error")
-            return redirect(url_for('watchlist.index'))
+        # If a "from_version" was requested, then find it (or the closest one)
+        # Also set "from version" to be the closest version to the one that was last viewed.
 
-        # Save the current newest history as the most recently viewed
-        datastore.set_last_viewed(uuid, time.time())
+        best_last_viewed_timestamp = watch.get_from_version_based_on_last_viewed
+        from_version_timestamp = best_last_viewed_timestamp if best_last_viewed_timestamp else dates[-2]
+        from_version = request.args.get('from_version', from_version_timestamp )
 
-        # Read as binary and force decode as UTF-8
-        # Windows may fail decode in python if we just use 'r' mode (chardet decode exception)
-        from_version = request.args.get('from_version')
-        from_version_index = -2  # second newest
-        if from_version and from_version in dates:
-            from_version_index = dates.index(from_version)
-        else:
-            from_version = dates[from_version_index]
+        # Use the current one if nothing was specified
+        to_version = request.args.get('to_version', str(dates[-1]))
 
         try:
-            from_version_file_contents = watch.get_history_snapshot(dates[from_version_index])
+            to_version_file_contents = watch.get_history_snapshot(timestamp=to_version)
         except Exception as e:
-            from_version_file_contents = f"Unable to read to-version at index {dates[from_version_index]}.\n"
-
-        to_version = request.args.get('to_version')
-        to_version_index = -1
-        if to_version and to_version in dates:
-            to_version_index = dates.index(to_version)
-        else:
-            to_version = dates[to_version_index]
+            logger.error(f"Unable to read watch history to-version for version {to_version}: {str(e)}")
+            to_version_file_contents = f"Unable to read to-version at {to_version}.\n"
 
         try:
-            to_version_file_contents = watch.get_history_snapshot(dates[to_version_index])
+            from_version_file_contents = watch.get_history_snapshot(timestamp=from_version)
         except Exception as e:
-            to_version_file_contents = "Unable to read to-version at index{}.\n".format(dates[to_version_index])
+            logger.error(f"Unable to read watch history from-version for version {from_version}: {str(e)}")
+            from_version_file_contents = f"Unable to read to-version {from_version}.\n"
 
         screenshot_url = watch.get_screenshot()
 
@@ -163,6 +168,8 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
         if datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False):
             password_enabled_and_share_is_off = not datastore.data['settings']['application'].get('shared_diff_access')
 
+        datastore.set_last_viewed(uuid, time.time())
+
         output = render_template("diff.html",
                                  current_diff_url=watch['url'],
                                  from_version=str(from_version),

changedetectionio/blueprint/watchlist/templates/watch-overview.html

@@ -4,6 +4,7 @@
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 <script>let nowtimeserver={{ now_time_server }};</script>
+<script>let favicon_baseURL="{{ url_for('static_content', group='favicon', filename="PLACEHOLDER")}}";</script>
 <script>
 // Initialize Feather icons after the page loads
 document.addEventListener('DOMContentLoaded', function() {
@@ -18,19 +19,20 @@ document.addEventListener('DOMContentLoaded', function() {
     transition: background-size 0.9s ease
 }
 </style>
-<div class="box">
+<div class="box" id="form-quick-watch-add">
 
     <form class="pure-form" action="{{ url_for('ui.ui_views.form_quick_watch_add', tag=active_tag_uuid) }}" method="POST" id="new-watch-form">
         <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
         <fieldset>
-            <legend>Add a new change detection watch</legend>
+            <legend>Add a new web page change detection watch</legend>
             <div id="watch-add-wrapper-zone">
-
                     {{ render_nolabel_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_nolabel_field(form.tags, value=active_tag.title if active_tag_uuid else '', placeholder="watch label / tag") }}
                     {{ render_nolabel_field(form.watch_submit_button, title="Watch this URL!" ) }}
                     {{ render_nolabel_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
             </div>
+            <div id="watch-group-tag">
+               {{ render_field(form.tags, value=active_tag.title if active_tag_uuid else '', placeholder="Watch group / tag", class="transparent-field") }}
+            </div>
             <div id="quick-watch-processor-type">
                 {{ render_simple_field(form.processor) }}
             </div>
@@ -38,7 +40,8 @@ document.addEventListener('DOMContentLoaded', function() {
         </fieldset>
         <span style="color:#eee; font-size: 80%;"><img alt="Create a shareable link" style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread-white.svg')}}" > Tip: You can also add 'shared' watches. <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Sharing-a-Watch">More info</a></span>
     </form>
-
+</div>
+<div class="box">
     <form class="pure-form" action="{{ url_for('ui.form_watch_list_checkbox_operations') }}" method="POST" id="watch-list-form">
     <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
     <input type="hidden" id="op_extradata" name="op_extradata" value="" >
@@ -80,14 +83,20 @@ document.addEventListener('DOMContentLoaded', function() {
     {%- endif -%}
 
     <div id="watch-table-wrapper">
-
-        <table class="pure-table pure-table-striped watch-table">
+        {%- set table_classes = [
+            'favicon-enabled' if  datastore.data['settings']['application']['ui'].get('favicons_enabled') else 'favicon-not-enabled',
+        ] -%}
+        <table class="pure-table pure-table-striped watch-table {{ table_classes | reject('equalto', '') | join(' ') }}">
             <thead>
             <tr>
                 {%- set link_order = "desc" if sort_order  == 'asc' else "asc" -%}
                 {%- set arrow_span = "" -%}
                 <th><input style="vertical-align: middle" type="checkbox" id="check-all" > <a class="{{ 'active '+link_order if sort_attribute == 'date_created' else 'inactive' }}"  href="{{url_for('watchlist.index', sort='date_created', order=link_order, tag=active_tag_uuid)}}"># <span class='arrow {{link_order}}'></span></a></th>
-                <th class="empty-cell"></th>
+                <th>
+                    <a class="{{ 'active '+link_order if sort_attribute == 'paused' else 'inactive' }}" href="{{url_for('watchlist.index', sort='paused', order=link_order, tag=active_tag_uuid)}}"><i data-feather="pause" style="vertical-align: bottom; width: 14px; height: 14px;  margin-right: 4px;"></i><span class='arrow {{link_order}}'></span></a>
+                    &nbsp;
+                    <a class="{{ 'active '+link_order if sort_attribute == 'notification_muted' else 'inactive' }}" href="{{url_for('watchlist.index', sort='notification_muted', order=link_order, tag=active_tag_uuid)}}"><i data-feather="volume-2" style="vertical-align: bottom; width: 14px; height: 14px;  margin-right: 4px;"></i><span class='arrow {{link_order}}'></span></a>
+                </th>
                 <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('watchlist.index', sort='label', order=link_order, tag=active_tag_uuid)}}">Website <span class='arrow {{link_order}}'></span></a></th>
              {%- if any_has_restock_price_processor -%}
                 <th>Restock &amp; Price</th>
@@ -103,9 +112,11 @@ document.addEventListener('DOMContentLoaded', function() {
                 <td colspan="{{ cols_required }}" style="text-wrap: wrap;">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('imports.import_page')}}" >import a list</a>.</td>
             </tr>
             {%- endif -%}
+
             {%- for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) -%}
                 {%- set checking_now = is_checking_now(watch) -%}
                 {%- set history_n = watch.history_n -%}
+                {%- set favicon = watch.get_favicon_filename() -%}
                 {#  Mirror in changedetectionio/static/js/realtime.js for the frontend #}
                 {%- set row_classes = [
                     loop.cycle('pure-table-odd', 'pure-table-even'),
@@ -114,49 +125,63 @@ document.addEventListener('DOMContentLoaded', function() {
                     'paused' if watch.paused is defined and watch.paused != False else '',
                     'unviewed' if watch.has_unviewed else '',
                     'has-restock-info' if watch.has_restock_info else 'no-restock-info',
+                    'has-favicon' if favicon else '',
                     'in-stock' if watch.has_restock_info and watch['restock']['in_stock'] else '',
                     'not-in-stock' if watch.has_restock_info and not watch['restock']['in_stock'] else '',
                     'queued' if watch.uuid in queued_uuids else '',
                     'checking-now' if checking_now else '',
                     'notification_muted' if watch.notification_muted else '',
                     'single-history' if history_n == 1 else '',
-                    'multiple-history' if history_n >= 2 else ''
+                    'multiple-history' if history_n >= 2 else '',                    
                 ] -%}
             <tr id="{{ watch.uuid }}" data-watch-uuid="{{ watch.uuid }}" class="{{ row_classes | reject('equalto', '') | join(' ') }}">
-                <td class="inline checkbox-uuid" ><input name="uuids"  type="checkbox" value="{{ watch.uuid}} " > <span>{{ loop.index+pagination.skip }}</span></td>
+                <td class="inline checkbox-uuid" ><div><input name="uuids"  type="checkbox" value="{{ watch.uuid}} " > <span class="counter-i">{{ loop.index+pagination.skip }}</span></div></td>
                 <td class="inline watch-controls">
+                    <div>
                     <a class="ajax-op state-off pause-toggle" data-op="pause" href="{{url_for('watchlist.index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause" ></a>
                     <a class="ajax-op state-on pause-toggle"  data-op="pause" style="display: none" href="{{url_for('watchlist.index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
                     <a class="ajax-op state-off mute-toggle" data-op="mute" href="{{url_for('watchlist.index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notification" title="Mute notification" class="icon icon-mute" ></a>
                     <a class="ajax-op state-on mute-toggle" data-op="mute"  style="display: none" href="{{url_for('watchlist.index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="UnMute notification" title="UnMute notification" class="icon icon-mute" ></a>
+                    </div>
                 </td>
-                <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
-                    <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}">&nbsp;</a>
-                    <a class="link-spread" href="{{url_for('ui.form_share_put_watch', uuid=watch.uuid)}}"><img src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="status-icon icon icon-spread" title="Create a link to share watch config with others" ></a>
 
-                    {%- if watch.get_fetch_backend == "html_webdriver"
-                         or ( watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
-                         or "extra_browser_" in watch.get_fetch_backend
-                    -%}
-                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='google-chrome-icon.png')}}" alt="Using a Chrome browser" title="Using a Chrome browser" >
-                    {%- endif -%}
+                <td class="title-col inline">
+                    <div class="flex-wrapper">
+                    {%  if datastore.data['settings']['application']['ui'].get('favicons_enabled') %}
+                        <div>{# A page might have hundreds of these images, set IMG options for lazy loading, don't set SRC if we dont have it so it doesnt fetch the placeholder'  #}
+                            <img alt="Favicon thumbnail" class="favicon" loading="lazy" decoding="async" fetchpriority="low" {% if favicon %} src="{{url_for('static_content', group='favicon', filename=watch.uuid)}}" {% else %} src='data:image/svg+xml;utf8,%3Csvg xmlns="http://www.w3.org/2000/svg" width="7.087" height="7.087" viewBox="0 0 7.087 7.087"%3E%3Ccircle cx="3.543" cy="3.543" r="3.279" stroke="%23e1e1e1" stroke-width="0.45" fill="none" opacity="0.74"/%3E%3C/svg%3E' {%  endif %} />
+                        </div>
+                    {%  endif %}
+                        <div>
+                        <span class="watch-title">
+                            {{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}&nbsp;<a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}">&nbsp;</a>
+                        </span>
+                            <div class="error-text" style="display:none;">{{ watch.compile_error_texts(has_proxies=datastore.proxy_list) }}</div>
+                            {%- if watch['processor'] == 'text_json_diff'  -%}
+                                {%- if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data']  -%}
+                                <div class="ldjson-price-track-offer">Switch to Restock & Price watch mode? <a href="{{url_for('price_data_follower.accept', uuid=watch.uuid)}}" class="pure-button button-xsmall">Yes</a> <a href="{{url_for('price_data_follower.reject', uuid=watch.uuid)}}" class="">No</a></div>
+                                {%- endif -%}
+                            {%- endif -%}
+                            {%- if watch['processor'] == 'restock_diff' -%}
+                                <span class="tracking-ldjson-price-data" title="Automatically following embedded price information"><img src="{{url_for('static_content', group='images', filename='price-tag-icon.svg')}}"  class="status-icon price-follow-tag-icon" > Price</span>
+                            {%- endif -%}
+                            {%- for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() -%}
+                              <span class="watch-tag-list">{{ watch_tag.title }}</span>
+                            {%- endfor -%}
+                        </div>
+                    <div class="status-icons">
+                            <a class="link-spread" href="{{url_for('ui.form_share_put_watch', uuid=watch.uuid)}}"><img src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="status-icon icon icon-spread" title="Create a link to share watch config with others" ></a>
+                            {%- if watch.get_fetch_backend == "html_webdriver"
+                                 or ( watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
+                                 or "extra_browser_" in watch.get_fetch_backend
+                            -%}
+                            <img class="status-icon" src="{{url_for('static_content', group='images', filename='google-chrome-icon.png')}}" alt="Using a Chrome browser" title="Using a Chrome browser" >
+                            {%- endif -%}
+                            {%- if watch.is_pdf  -%}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" alt="Converting PDF to text" >{%- endif -%}
+                            {%- if watch.has_browser_steps -%}<img class="status-icon status-browsersteps" src="{{url_for('static_content', group='images', filename='steps.svg')}}" alt="Browser Steps is enabled" >{%- endif -%}
 
-                    {%- if watch.is_pdf  -%}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" alt="Converting PDF to text" >{%- endif -%}
-                    {%- if watch.has_browser_steps -%}<img class="status-icon status-browsersteps" src="{{url_for('static_content', group='images', filename='steps.svg')}}" alt="Browser Steps is enabled" >{%- endif -%}
-
-                    <div class="error-text" style="display:none;">{{ watch.compile_error_texts(has_proxies=datastore.proxy_list)|safe }}</div>
-
-                    {%- if watch['processor'] == 'text_json_diff'  -%}
-                        {%- if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data']  -%}
-                        <div class="ldjson-price-track-offer">Switch to Restock & Price watch mode? <a href="{{url_for('price_data_follower.accept', uuid=watch.uuid)}}" class="pure-button button-xsmall">Yes</a> <a href="{{url_for('price_data_follower.reject', uuid=watch.uuid)}}" class="">No</a></div>
-                        {%- endif -%}
-                    {%- endif -%}
-                    {%- if watch['processor'] == 'restock_diff' -%}
-                        <span class="tracking-ldjson-price-data" title="Automatically following embedded price information"><img src="{{url_for('static_content', group='images', filename='price-tag-icon.svg')}}"  class="status-icon price-follow-tag-icon" > Price</span>
-                    {%- endif -%}
-                    {%- for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() -%}
-                      <span class="watch-tag-list">{{ watch_tag.title }}</span>
-                    {%- endfor -%}
+                    </div>
+                    </div>
                 </td>
 {%- if any_has_restock_price_processor -%}
                 <td class="restock-and-price">
@@ -193,13 +218,15 @@ document.addEventListener('DOMContentLoaded', function() {
                     Not yet
                     {%- endif -%}
                 </td>
-                <td>
+                <td class="buttons">
+                    <div>
                     {%- set target_attr = ' target="' ~ watch.uuid ~ '"' if datastore.data['settings']['application']['ui'].get('open_diff_in_new_tab') else '' -%}
                     <a href="" class="already-in-queue-button recheck pure-button pure-button-primary" style="display: none;" disabled="disabled">Queued</a>
                     <a href="{{ url_for('ui.form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}" data-op='recheck' class="ajax-op recheck pure-button pure-button-primary">Recheck</a>
                     <a href="{{ url_for('ui.ui_edit.edit_page', uuid=watch.uuid, tag=active_tag_uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
                     <a href="{{ url_for('ui.ui_views.diff_history_page', uuid=watch.uuid)}}" {{target_attr}} class="pure-button pure-button-primary history-link" style="display: none;">History</a>
                     <a href="{{ url_for('ui.ui_views.preview_page', uuid=watch.uuid)}}" {{target_attr}} class="pure-button pure-button-primary preview-link" style="display: none;">Preview</a>
+                    </div>
                 </td>
             </tr>
             {%- endfor -%}
@@ -212,9 +239,14 @@ document.addEventListener('DOMContentLoaded', function() {
             <li id="post-list-mark-views" class="{%- if has_unviewed -%}has-unviewed{%- endif -%}" style="display: none;" >
                 <a href="{{url_for('ui.mark_all_viewed',with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag " id="mark-all-viewed">Mark all viewed</a>
             </li>
+        {%-  if active_tag_uuid -%}
+            <li id="post-list-mark-views-tag">
+                <a href="{{url_for('ui.mark_all_viewed', tag=active_tag_uuid) }}" class="pure-button button-tag " id="mark-all-viewed">Mark all viewed in '{{active_tag.title}}'</a>
+            </li>
+        {%-  endif -%}
             <li>
                <a href="{{ url_for('ui.form_watch_checknow', tag=active_tag_uuid, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag" id="recheck-all">Recheck
-                all {%- if active_tag_uuid-%}  in "{{active_tag.title}}"{%endif%}</a>
+                all {% if active_tag_uuid %}  in '{{active_tag.title}}'{%endif%}</a>
             </li>
             <li>
                 <a href="{{ url_for('rss.feed', tag=active_tag_uuid, token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='generic_feed-icon.svg')}}" height="15"></a>

changedetectionio/conditions/__init__.py

@@ -1,5 +1,3 @@
-from flask import Blueprint
-
 from json_logic.builtins import BUILTINS
 
 from .exceptions import EmptyConditionRuleRowNotUsable
@@ -16,7 +14,6 @@ operator_choices = [
     ("==", "Equals"),
     ("!=", "Not Equals"),
     ("in", "Contains"),
-    ("!in", "Does Not Contain"),
 ]
 
 # Fields available in the rules

changedetectionio/conditions/default_plugin.py

@@ -21,17 +21,21 @@ def register_operators():
     def length_max(_, text, strlen):
         return len(text) <= int(strlen)
 
-    # ✅ Custom function for case-insensitive regex matching
+    # Custom function for case-insensitive regex matching
     def contains_regex(_, text, pattern):
         """Returns True if `text` contains `pattern` (case-insensitive regex match)."""
         return bool(re.search(pattern, str(text), re.IGNORECASE))
 
-    # ✅ Custom function for NOT matching case-insensitive regex
+    # Custom function for NOT matching case-insensitive regex
     def not_contains_regex(_, text, pattern):
         """Returns True if `text` does NOT contain `pattern` (case-insensitive regex match)."""
         return not bool(re.search(pattern, str(text), re.IGNORECASE))
 
+    def not_contains(_, text, pattern):
+        return not pattern in text
+
     return {
+        "!in": not_contains,
         "!contains_regex": not_contains_regex,
         "contains_regex": contains_regex,
         "ends_with": ends_with,
@@ -43,6 +47,7 @@ def register_operators():
 @hookimpl
 def register_operator_choices():
     return [
+        ("!in", "Does NOT Contain"),
         ("starts_with", "Text Starts With"),
         ("ends_with", "Text Ends With"),
         ("length_min", "Length minimum"),

changedetectionio/conditions/plugins/levenshtein_plugin.py

@@ -1,6 +1,8 @@
 import pluggy
 from loguru import logger
 
+LEVENSHTEIN_MAX_LEN_FOR_EDIT_STATS=100000
+
 # Support both plugin systems
 conditions_hookimpl = pluggy.HookimplMarker("changedetectionio_conditions")
 global_hookimpl = pluggy.HookimplMarker("changedetectionio")
@@ -72,7 +74,17 @@ def ui_edit_stats_extras(watch):
     """Generate the HTML for Levenshtein stats - shared by both plugin systems"""
     if len(watch.history.keys()) < 2:
         return "<p>Not enough history to calculate Levenshtein metrics</p>"
-    
+
+
+    # Protection against the algorithm getting stuck on huge documents
+    k = list(watch.history.keys())
+    if any(
+            len(watch.get_history_snapshot(timestamp=k[idx])) > LEVENSHTEIN_MAX_LEN_FOR_EDIT_STATS
+            for idx in (-1, -2)
+            if len(k) >= abs(idx)
+    ):
+        return "<p>Snapshot too large for edit statistics, skipping.</p>"
+
     try:
         lev_data = levenshtein_ratio_recent_history(watch)
         if not lev_data or not isinstance(lev_data, dict):

changedetectionio/content_fetchers/__init__.py

@@ -28,6 +28,7 @@ from changedetectionio.content_fetchers.requests import fetcher as html_requests
 import importlib.resources
 XPATH_ELEMENT_JS = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text(encoding='utf-8')
 INSTOCK_DATA_JS = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('stock-not-in-stock.js').read_text(encoding='utf-8')
+FAVICON_FETCHER_JS = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('favicon-fetcher.js').read_text(encoding='utf-8')
 
 
 def available_fetchers():

changedetectionio/content_fetchers/base.py

@@ -48,6 +48,7 @@ class Fetcher():
     error = None
     fetcher_description = "No description"
     headers = {}
+    favicon_blob = None
     instock_data = None
     instock_data_js = ""
     status_code = None

changedetectionio/content_fetchers/playwright.py

@@ -5,7 +5,7 @@ from urllib.parse import urlparse
 from loguru import logger
 
 from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
-    SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_MAX_TOTAL_HEIGHT, XPATH_ELEMENT_JS, INSTOCK_DATA_JS
+    SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_MAX_TOTAL_HEIGHT, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, FAVICON_FETCHER_JS
 from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable
 
@@ -234,6 +234,12 @@ class fetcher(Fetcher):
                 await browser.close()
                 raise PageUnloadable(url=url, status_code=None, message=str(e))
 
+            try:
+                self.favicon_blob = await self.page.evaluate(FAVICON_FETCHER_JS)
+                await self.page.request_gc()
+            except Exception as e:
+                logger.error(f"Error fetching FavIcon info {str(e)}, continuing.")
+
             if self.status_code != 200 and not ignore_status_codes:
                 screenshot = await capture_full_page_async(self.page)
                 raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
@@ -274,6 +280,7 @@ class fetcher(Fetcher):
             await self.page.request_gc()
             logger.debug(f"Scrape xPath element data in browser done in {time.time() - now:.2f}s")
 
+
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
             # JPEG is better here because the screenshots can be very very large

changedetectionio/content_fetchers/puppeteer.py

@@ -8,7 +8,7 @@ from loguru import logger
 
 from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
     SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_DEFAULT_QUALITY, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, \
-    SCREENSHOT_MAX_TOTAL_HEIGHT
+    SCREENSHOT_MAX_TOTAL_HEIGHT, FAVICON_FETCHER_JS
 from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, \
     BrowserConnectError
@@ -51,7 +51,15 @@ async def capture_full_page(page):
         await page.setViewport({'width': page.viewport['width'], 'height': step_size})
 
     while y < min(page_height, SCREENSHOT_MAX_TOTAL_HEIGHT):
-        await page.evaluate(f"window.scrollTo(0, {y})")
+        # better than scrollTo incase they override it in the page
+        await page.evaluate(
+            """(y) => {
+                document.documentElement.scrollTop = y;
+                document.body.scrollTop = y;
+            }""",
+            y
+        )
+
         screenshot_chunks.append(await page.screenshot(type_='jpeg',
                                                        fullPage=False,
                                                        quality=int(os.getenv("SCREENSHOT_QUALITY", 72))))
@@ -149,7 +157,11 @@ class fetcher(Fetcher):
                          ):
         import re
         self.delete_browser_steps_screenshots()
-        extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
+
+        n = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
+        extra_wait = min(n, 15)
+
+        logger.debug(f"Extra wait set to {extra_wait}s, requested was {n}s.")
 
         from pyppeteer import Pyppeteer
         pyppeteer_instance = Pyppeteer()
@@ -165,12 +177,10 @@ class fetcher(Fetcher):
         except websockets.exceptions.InvalidURI:
             raise BrowserConnectError(msg=f"Error connecting to the browser, check your browser connection address (should be ws:// or wss://")
         except Exception as e:
-            raise BrowserConnectError(msg=f"Error connecting to the browser {str(e)}")
+            raise BrowserConnectError(msg=f"Error connecting to the browser - Exception '{str(e)}'")
 
-        # Better is to launch chrome with the URL as arg
-        # non-headless - newPage() will launch an extra tab/window, .browser should already contain 1 page/tab
-        # headless - ask a new page
-        self.page = (pages := await browser.pages) and len(pages) or await browser.newPage()
+        # more reliable is to just request a new page
+        self.page = await browser.newPage()
 
         if '--window-size' in self.browser_connection_url:
             # Be sure the viewport is always the window-size, this is often not the same thing
@@ -227,13 +237,35 @@ class fetcher(Fetcher):
         #            browsersteps_interface = steppable_browser_interface()
         #            browsersteps_interface.page = self.page
 
-        response = await self.page.goto(url, waitUntil="load")
+        async def handle_frame_navigation(event):
+            logger.debug(f"Frame navigated: {event}")
+            w = extra_wait - 2 if extra_wait > 4 else 2
+            logger.debug(f"Waiting {w} seconds before calling Page.stopLoading...")
+            await asyncio.sleep(w)
+            logger.debug("Issuing stopLoading command...")
+            await self.page._client.send('Page.stopLoading')
+            logger.debug("stopLoading command sent!")
 
-        if response is None:
-            await self.page.close()
-            await browser.close()
-            logger.warning("Content Fetcher > Response object was none (as in, the response from the browser was empty, not just the content)")
-            raise EmptyReply(url=url, status_code=None)
+        self.page._client.on('Page.frameStartedNavigating', lambda event: asyncio.create_task(handle_frame_navigation(event)))
+        self.page._client.on('Page.frameStartedLoading', lambda event: asyncio.create_task(handle_frame_navigation(event)))
+        self.page._client.on('Page.frameStoppedLoading', lambda event: logger.debug(f"Frame stopped loading: {event}"))
+
+        response = None
+        attempt=0
+        while not response:
+            logger.debug(f"Attempting page fetch {url} attempt {attempt}")
+            response = await self.page.goto(url)
+            await asyncio.sleep(1 + extra_wait)
+            if response:
+                break
+            if not response:
+                logger.warning("Page did not fetch! trying again!")
+            if response is None and attempt>=2:
+                await self.page.close()
+                await browser.close()
+                logger.warning(f"Content Fetcher > Response object was none (as in, the response from the browser was empty, not just the content) exiting attmpt {attempt}")
+                raise EmptyReply(url=url, status_code=None)
+            attempt+=1
 
         self.headers = response.headers
 
@@ -258,6 +290,11 @@ class fetcher(Fetcher):
             await browser.close()
             raise PageUnloadable(url=url, status_code=None, message=str(e))
 
+        try:
+            self.favicon_blob = await self.page.evaluate(FAVICON_FETCHER_JS)
+        except Exception as e:
+            logger.error(f"Error fetching FavIcon info {str(e)}, continuing.")
+
         if self.status_code != 200 and not ignore_status_codes:
             screenshot = await capture_full_page(page=self.page)
 
@@ -276,7 +313,6 @@ class fetcher(Fetcher):
         #            if self.browser_steps_get_valid_steps():
         #                self.iterate_browser_steps()
 
-        await asyncio.sleep(1 + extra_wait)
 
         # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
         # Setup the xPath/VisualSelector scraper

changedetectionio/content_fetchers/res/favicon-fetcher.js

@@ -0,0 +1,79 @@
+(async () => {
+  const links = Array.from(document.querySelectorAll(
+    'link[rel~="apple-touch-icon"], link[rel~="icon"]'
+  ));
+
+  const icons = links.map(link => {
+    const sizesStr = link.getAttribute('sizes');
+    let size = 0;
+    if (sizesStr) {
+      const [w] = sizesStr.split('x').map(Number);
+      if (!isNaN(w)) size = w;
+    } else {
+      size = 16;
+    }
+    return {
+      size,
+      rel: link.getAttribute('rel'),
+      href: link.href
+    };
+  });
+
+  // If no icons found, add fallback favicon.ico
+  if (icons.length === 0) {
+    icons.push({
+      size: 16,
+      rel: 'icon',
+      href: '/favicon.ico'
+    });
+  }
+
+  // sort preference
+  icons.sort((a, b) => {
+    const isAppleA = /apple-touch-icon/.test(a.rel);
+    const isAppleB = /apple-touch-icon/.test(b.rel);
+    if (isAppleA && !isAppleB) return -1;
+    if (!isAppleA && isAppleB) return 1;
+    return b.size - a.size;
+  });
+
+  const timeoutMs = 2000;
+
+  for (const icon of icons) {
+    try {
+      const controller = new AbortController();
+      const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+      const resp = await fetch(icon.href, {
+        signal: controller.signal,
+        redirect: 'follow'
+      });
+
+      clearTimeout(timeout);
+
+      if (!resp.ok) {
+        continue;
+      }
+
+      const blob = await resp.blob();
+
+      // Convert blob to base64
+      const reader = new FileReader();
+      return await new Promise(resolve => {
+        reader.onloadend = () => {
+          resolve({
+            url: icon.href,
+            base64: reader.result.split(",")[1]
+          });
+        };
+        reader.readAsDataURL(blob);
+      });
+
+    } catch (e) {
+      continue;
+    }
+  }
+
+  // nothing found
+  return null;
+})();

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -17,7 +17,9 @@ async () => {
             'back in stock soon',
             'back-order or out of stock',
             'backordered',
+            'backorder',
             'benachrichtigt mich', // notify me
+            'binnenkort leverbaar', // coming soon
             'brak na stanie',
             'brak w magazynie',
             'coming soon',
@@ -38,6 +40,7 @@ async () => {
             'mail me when available',
             'message if back in stock',
             'mevcut değil',
+            'more on order',
             'nachricht bei',
             'nicht auf lager',
             'nicht lagernd',
@@ -85,6 +88,7 @@ async () => {
             'tidak tersedia',
             'tijdelijk uitverkocht',
             'tiket tidak tersedia',
+            'to subscribe to back in stock',
             'tükendi',
             'unavailable nearby',
             'unavailable tickets',
@@ -119,8 +123,7 @@ async () => {
             return text.toLowerCase().trim();
         }
 
-        const negateOutOfStockRegex = new RegExp('^([0-9] in stock|add to cart|in stock)', 'ig');
-
+        const negateOutOfStockRegex = new RegExp('^([0-9] in stock|add to cart|in stock|arrives approximately)', 'ig');
         // The out-of-stock or in-stock-text is generally always above-the-fold
         // and often below-the-fold is a list of related products that may or may not contain trigger text
         // so it's good to filter to just the 'above the fold' elements

changedetectionio/custom_queue.py

@@ -3,6 +3,41 @@ import asyncio
 from blinker import signal
 from loguru import logger
 
+
+class NotificationQueue(queue.Queue):
+    """
+    Extended Queue that sends a 'notification_event' signal when notifications are added.
+    
+    This class extends the standard Queue and adds a signal emission after a notification
+    is put into the queue. The signal includes the watch UUID if available.
+    """
+    
+    def __init__(self, maxsize=0):
+        super().__init__(maxsize)
+        try:
+            self.notification_event_signal = signal('notification_event')
+        except Exception as e:
+            logger.critical(f"Exception creating notification_event signal: {e}")
+
+    def put(self, item, block=True, timeout=None):
+        # Call the parent's put method first
+        super().put(item, block, timeout)
+        
+        # After putting the notification in the queue, emit signal with watch UUID
+        try:
+            if self.notification_event_signal and isinstance(item, dict):
+                watch_uuid = item.get('uuid')
+                if watch_uuid:
+                    # Send the notification_event signal with the watch UUID
+                    self.notification_event_signal.send(watch_uuid=watch_uuid)
+                    logger.trace(f"NotificationQueue: Emitted notification_event signal for watch UUID {watch_uuid}")
+                else:
+                    # Send signal without UUID for system notifications
+                    self.notification_event_signal.send()
+                    logger.trace("NotificationQueue: Emitted notification_event signal for system notification")
+        except Exception as e:
+            logger.error(f"Exception emitting notification_event signal: {e}")
+
 class SignalPriorityQueue(queue.PriorityQueue):
     """
     Extended PriorityQueue that sends a signal when items with a UUID are added.

changedetectionio/flask_app.py

@@ -12,19 +12,17 @@ from blinker import signal
 
 from changedetectionio.strtobool import strtobool
 from threading import Event
-from changedetectionio.custom_queue import SignalPriorityQueue, AsyncSignalPriorityQueue
+from changedetectionio.custom_queue import SignalPriorityQueue, AsyncSignalPriorityQueue, NotificationQueue
 from changedetectionio import worker_handler
 
 from flask import (
     Flask,
     abort,
     flash,
-    make_response,
     redirect,
     render_template,
     request,
     send_from_directory,
-    session,
     url_for,
 )
 from flask_compress import Compress as FlaskCompress
@@ -40,7 +38,7 @@ from loguru import logger
 
 from changedetectionio import __version__
 from changedetectionio import queuedWatchMetaData
-from changedetectionio.api import Watch, WatchHistory, WatchSingleHistory, CreateWatch, Import, SystemInfo, Tag, Tags, Notifications
+from changedetectionio.api import Watch, WatchHistory, WatchSingleHistory, CreateWatch, Import, SystemInfo, Tag, Tags, Notifications, WatchFavicon
 from changedetectionio.api.Search import Search
 from .time_handler import is_within_schedule
 
@@ -52,7 +50,7 @@ extra_stylesheets = []
 
 # Use async queue by default, keep sync for backward compatibility  
 update_q = AsyncSignalPriorityQueue() if worker_handler.USE_ASYNC_WORKERS else SignalPriorityQueue()
-notification_q = queue.Queue()
+notification_q = NotificationQueue()
 MAX_QUEUE_SIZE = 2000
 
 app = Flask(__name__,
@@ -100,7 +98,7 @@ watch_api = Api(app, decorators=[csrf.exempt])
 def init_app_secret(datastore_path):
     secret = ""
 
-    path = "{}/secret.txt".format(datastore_path)
+    path = os.path.join(datastore_path, "secret.txt")
 
     try:
         with open(path, "r") as f:
@@ -307,7 +305,9 @@ def changedetection_app(config=None, datastore_o=None):
     watch_api.add_resource(WatchSingleHistory,
                            '/api/v1/watch/<string:uuid>/history/<string:timestamp>',
                            resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
-
+    watch_api.add_resource(WatchFavicon,
+                           '/api/v1/watch/<string:uuid>/favicon',
+                           resource_class_kwargs={'datastore': datastore})
     watch_api.add_resource(WatchHistory,
                            '/api/v1/watch/<string:uuid>/history',
                            resource_class_kwargs={'datastore': datastore})
@@ -427,6 +427,32 @@ def changedetection_app(config=None, datastore_o=None):
             except FileNotFoundError:
                 abort(404)
 
+        if group == 'favicon':
+            # Could be sensitive, follow password requirements
+            if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated:
+                abort(403)
+            # Get the watch object
+            watch = datastore.data['watching'].get(filename)
+            if not watch:
+                abort(404)
+
+            favicon_filename = watch.get_favicon_filename()
+            if favicon_filename:
+                try:
+                    import magic
+                    mime = magic.from_file(
+                        os.path.join(watch.watch_data_dir, favicon_filename),
+                        mime=True
+                    )
+                except ImportError:
+                    # Fallback, no python-magic
+                    import mimetypes
+                    mime, encoding = mimetypes.guess_type(favicon_filename)
+
+                response = make_response(send_from_directory(watch.watch_data_dir, favicon_filename))
+                response.headers['Content-type'] = mime
+                response.headers['Cache-Control'] = 'max-age=300, must-revalidate'  # Cache for 5 minutes, then revalidate
+                return response
 
         if group == 'visual_selector_data':
             # Could be sensitive, follow password requirements

changedetectionio/forms.py

@@ -738,8 +738,9 @@ class globalSettingsRequestForm(Form):
                     return False
 
 class globalSettingsApplicationUIForm(Form):
-    open_diff_in_new_tab = BooleanField('Open diff page in a new tab', default=True, validators=[validators.Optional()])
+    open_diff_in_new_tab = BooleanField("Open 'History' page in a new tab", default=True, validators=[validators.Optional()])
     socket_io_enabled = BooleanField('Realtime UI Updates Enabled', default=True, validators=[validators.Optional()])
+    favicons_enabled = BooleanField('Favicons Enabled', default=True, validators=[validators.Optional()])
 
 # datastore.data['settings']['application']..
 class globalSettingsApplicationForm(commonSettingsForm):

changedetectionio/model/App.py

@@ -62,7 +62,8 @@ class model(dict):
                     'timezone': None, # Default IANA timezone name
                     'ui': {
                         'open_diff_in_new_tab': True,
-                        'socket_io_enabled': True
+                        'socket_io_enabled': True,
+                        'favicons_enabled': True
                     },
                 }
             }

changedetectionio/model/Watch.py

@@ -8,6 +8,7 @@ import re
 from pathlib import Path
 from loguru import logger
 
+from .. import safe_jinja
 from ..html_tools import TRANSLATE_WHITESPACE_TABLE
 
 # Allowable protocols, protects against javascript: etc
@@ -102,6 +103,13 @@ class model(watch_base):
             return 'DISABLED'
         return ready_url
 
+    @property
+    def domain_only_from_link(self):
+        from urllib.parse import urlparse
+        parsed = urlparse(self.link)
+        domain = parsed.hostname
+        return domain
+
     def clear_watch(self):
         import pathlib
 
@@ -412,6 +420,132 @@ class model(watch_base):
         # False is not an option for AppRise, must be type None
         return None
 
+    def bump_favicon(self, url, favicon_base_64: str) -> None:
+        from urllib.parse import urlparse
+        import base64
+        import binascii
+        decoded = None
+
+        if url:
+            try:
+                parsed = urlparse(url)
+                filename = os.path.basename(parsed.path)
+                (base, extension) = filename.lower().strip().rsplit('.', 1)
+            except ValueError:
+                logger.error(f"UUID: {self.get('uuid')} Cant work out file extension from '{url}'")
+                return None
+        else:
+            # Assume favicon.ico
+            base = "favicon"
+            extension = "ico"
+
+        fname = os.path.join(self.watch_data_dir, f"favicon.{extension}")
+
+        try:
+            # validate=True makes sure the string only contains valid base64 chars
+            decoded = base64.b64decode(favicon_base_64, validate=True)
+        except (binascii.Error, ValueError) as e:
+            logger.warning(f"UUID: {self.get('uuid')} FavIcon save data (Base64) corrupt? {str(e)}")
+        else:
+            if decoded:
+                try:
+                    with open(fname, 'wb') as f:
+                        f.write(decoded)
+                    # A signal that could trigger the socket server to update the browser also
+                    watch_check_update = signal('watch_favicon_bump')
+                    if watch_check_update:
+                        watch_check_update.send(watch_uuid=self.get('uuid'))
+
+                except Exception as e:
+                    logger.warning(f"UUID: {self.get('uuid')} error saving FavIcon to {fname} - {str(e)}")
+
+        # @todo - Store some checksum and only write when its different
+        logger.debug(f"UUID: {self.get('uuid')} updated favicon to at {fname}")
+
+    def get_favicon_filename(self) -> str | None:
+        """
+        Find any favicon.* file in the current working directory
+        and return the contents of the newest one.
+
+        Returns:
+            bytes: Contents of the newest favicon file, or None if not found.
+        """
+        import glob
+
+        # Search for all favicon.* files
+        files = glob.glob(os.path.join(self.watch_data_dir, "favicon.*"))
+
+        if not files:
+            return None
+
+        # Find the newest by modification time
+        newest_file = max(files, key=os.path.getmtime)
+        return os.path.basename(newest_file)
+
+    def get_screenshot_as_thumbnail(self, max_age=3200):
+        """Return path to a square thumbnail of the most recent screenshot.
+
+        Creates a 150x150 pixel thumbnail from the top portion of the screenshot.
+
+        Args:
+            max_age: Maximum age in seconds before recreating thumbnail
+
+        Returns:
+            Path to thumbnail or None if no screenshot exists
+        """
+        import os
+        import time
+
+        thumbnail_path = os.path.join(self.watch_data_dir, "thumbnail.jpeg")
+        top_trim = 500  # Pixels from top of screenshot to use
+
+        screenshot_path = self.get_screenshot()
+        if not screenshot_path:
+            return None
+
+        # Reuse thumbnail if it's fresh and screenshot hasn't changed
+        if os.path.isfile(thumbnail_path):
+            thumbnail_mtime = os.path.getmtime(thumbnail_path)
+            screenshot_mtime = os.path.getmtime(screenshot_path)
+
+            if screenshot_mtime <= thumbnail_mtime and time.time() - thumbnail_mtime < max_age:
+                return thumbnail_path
+
+        try:
+            from PIL import Image
+
+            with Image.open(screenshot_path) as img:
+                # Crop top portion first (full width, top_trim height)
+                top_crop_height = min(top_trim, img.height)
+                img = img.crop((0, 0, img.width, top_crop_height))
+
+                # Create a smaller intermediate image (to reduce memory usage)
+                aspect = img.width / img.height
+                interim_width = min(top_trim, img.width)
+                interim_height = int(interim_width / aspect) if aspect > 0 else top_trim
+                img = img.resize((interim_width, interim_height), Image.NEAREST)
+
+                # Convert to RGB if needed
+                if img.mode != 'RGB':
+                    img = img.convert('RGB')
+
+                # Crop to square from top center
+                square_size = min(img.width, img.height)
+                left = (img.width - square_size) // 2
+                img = img.crop((left, 0, left + square_size, square_size))
+
+                # Final resize to exact thumbnail size with better filter
+                img = img.resize((350, 350), Image.BILINEAR)
+
+                # Save with optimized settings
+                img.save(thumbnail_path, "JPEG", quality=75, optimize=True)
+
+            return thumbnail_path
+
+        except Exception as e:
+            logger.error(f"Error creating thumbnail for {self.get('uuid')}: {str(e)}")
+            return None
+
     def __get_file_ctime(self, filename):
         fname = os.path.join(self.watch_data_dir, filename)
         if os.path.isfile(fname):
@@ -691,11 +825,11 @@ class model(watch_base):
                 output.append(str(Markup(f"<div class=\"notification-error\"><a href=\"{url_for('settings.notification_logs')}\">{ self.get('last_notification_error') }</a></div>")))
 
         else:
-            # Lo_Fi version
+            # Lo_Fi version - no app context, cant rely on Jinja2 Markup
             if last_error:
-                output.append(str(Markup(last_error)))
+                output.append(safe_jinja.render_fully_escaped(last_error))
             if self.get('last_notification_error'):
-                output.append(str(Markup(self.get('last_notification_error'))))
+                output.append(safe_jinja.render_fully_escaped(self.get('last_notification_error')))
 
         res = "\n".join(output)
         return res

changedetectionio/realtime/socket_server.py

@@ -29,10 +29,18 @@ class SignalHandler:
         watch_delete_signal = signal('watch_deleted')
         watch_delete_signal.connect(self.handle_deleted_signal, weak=False)
 
+        watch_favicon_bumped_signal = signal('watch_favicon_bump')
+        watch_favicon_bumped_signal.connect(self.handle_watch_bumped_favicon_signal, weak=False)
+
+        # Connect to the notification_event signal
+        notification_event_signal = signal('notification_event')
+        notification_event_signal.connect(self.handle_notification_event, weak=False)
+        logger.info("SignalHandler: Connected to notification_event signal")
+
         # Create and start the queue update thread using standard threading
         import threading
         self.polling_emitter_thread = threading.Thread(
-            target=self.polling_emit_running_or_queued_watches_threaded, 
+            target=self.polling_emit_running_or_queued_watches_threaded,
             daemon=True
         )
         self.polling_emitter_thread.start()
@@ -64,6 +72,16 @@ class SignalHandler:
             else:
                 logger.warning(f"Watch UUID {watch_uuid} not found in datastore")
 
+    def handle_watch_bumped_favicon_signal(self, *args, **kwargs):
+        watch_uuid = kwargs.get('watch_uuid')
+        if watch_uuid:
+            # Emit the queue size to all connected clients
+            self.socketio_instance.emit("watch_bumped_favicon", {
+                "uuid": watch_uuid,
+                "event_timestamp": time.time()
+            })
+        logger.debug(f"Watch UUID {watch_uuid} got its favicon updated")
+
     def handle_deleted_signal(self, *args, **kwargs):
         watch_uuid = kwargs.get('watch_uuid')
         if watch_uuid:
@@ -89,33 +107,49 @@ class SignalHandler:
         except Exception as e:
             logger.error(f"Socket.IO error in handle_queue_length: {str(e)}")
 
+    def handle_notification_event(self, *args, **kwargs):
+        """Handle notification_event signal and emit to all clients"""
+        try:
+            watch_uuid = kwargs.get('watch_uuid')
+            logger.debug(f"SignalHandler: Notification event received for watch UUID: {watch_uuid}")
+
+            # Emit the notification event to all connected clients
+            self.socketio_instance.emit("notification_event", {
+                "watch_uuid": watch_uuid,
+                "event_timestamp": time.time()
+            })
+
+            logger.trace(f"Socket.IO: Emitted notification_event for watch UUID {watch_uuid}")
+
+        except Exception as e:
+            logger.error(f"Socket.IO error in handle_notification_event: {str(e)}")
 
     def polling_emit_running_or_queued_watches_threaded(self):
         """Threading version of polling for Windows compatibility"""
         import time
         import threading
         logger.info("Queue update thread started (threading mode)")
-        
+
         # Import here to avoid circular imports
         from changedetectionio.flask_app import app
         from changedetectionio import worker_handler
         watch_check_update = signal('watch_check_update')
-        
+
         # Track previous state to avoid unnecessary emissions
         previous_running_uuids = set()
-        
+
         # Run until app shutdown - check exit flag more frequently for fast shutdown
         exit_event = getattr(app.config, 'exit', threading.Event())
-        
+
         while not exit_event.is_set():
             try:
                 # Get current running UUIDs from async workers
                 running_uuids = set(worker_handler.get_running_uuids())
-                
+
                 # Only send updates for UUIDs that changed state
                 newly_running = running_uuids - previous_running_uuids
                 no_longer_running = previous_running_uuids - running_uuids
-                
+
                 # Send updates for newly running UUIDs (but exit fast if shutdown requested)
                 for uuid in newly_running:
                     if exit_event.is_set():
@@ -124,7 +158,7 @@ class SignalHandler:
                     with app.app_context():
                         watch_check_update.send(app_context=app, watch_uuid=uuid)
                     time.sleep(0.01)  # Small yield
-                
+
                 # Send updates for UUIDs that finished processing (but exit fast if shutdown requested)
                 if not exit_event.is_set():
                     for uuid in no_longer_running:
@@ -134,16 +168,16 @@ class SignalHandler:
                         with app.app_context():
                             watch_check_update.send(app_context=app, watch_uuid=uuid)
                         time.sleep(0.01)  # Small yield
-                
+
                 # Update tracking for next iteration
                 previous_running_uuids = running_uuids
-                
+
                 # Sleep between polling cycles, but check exit flag every 0.5 seconds for fast shutdown
                 for _ in range(20):  # 20 * 0.5 = 10 seconds total
                     if exit_event.is_set():
                         break
                     time.sleep(0.5)
-                
+
             except Exception as e:
                 logger.error(f"Error in threading polling: {str(e)}")
                 # Even during error recovery, check for exit quickly
@@ -151,11 +185,11 @@ class SignalHandler:
                     if exit_event.is_set():
                         break
                     time.sleep(0.5)
-        
+
         # Check if we're in pytest environment - if so, be more gentle with logging
         import sys
         in_pytest = "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ
-        
+
         if not in_pytest:
             logger.info("Queue update thread stopped (threading mode)")
 
@@ -186,20 +220,20 @@ def handle_watch_update(socketio, **kwargs):
 
         watch_data = {
             'checking_now': True if watch.get('uuid') in running_uuids else False,
+            'error_text': error_texts,
+            'event_timestamp': time.time(),
             'fetch_time': watch.get('fetch_time'),
             'has_error': True if error_texts else False,
-            'last_changed': watch.get('last_changed'),
-            'last_checked': watch.get('last_checked'),
-            'error_text': error_texts,
+            'has_favicon': True if watch.get_favicon_filename() else False,
             'history_n': watch.history_n,
-            'last_checked_text': _jinja2_filter_datetime(watch),
             'last_changed_text': timeago.format(int(watch.last_changed), time.time()) if watch.history_n >= 2 and int(watch.last_changed) > 0 else 'Not yet',
-            'queued': True if watch.get('uuid') in queue_list else False,
-            'paused': True if watch.get('paused') else False,
+            'last_checked': watch.get('last_checked'),
+            'last_checked_text': _jinja2_filter_datetime(watch),
             'notification_muted': True if watch.get('notification_muted') else False,
+            'paused': True if watch.get('paused') else False,
+            'queued': True if watch.get('uuid') in queue_list else False,
             'unviewed': watch.has_unviewed,
             'uuid': watch.get('uuid'),
-            'event_timestamp': time.time()
         }
 
         errored_count = 0
@@ -229,15 +263,15 @@ def init_socketio(app, datastore):
     """Initialize SocketIO with the main Flask app"""
     import platform
     import sys
-    
+
     # Platform-specific async_mode selection for better stability
     system = platform.system().lower()
     python_version = sys.version_info
-    
+
     # Check for SocketIO mode configuration via environment variable
     # Default is 'threading' for best cross-platform compatibility
     socketio_mode = os.getenv('SOCKETIO_MODE', 'threading').lower()
-    
+
     if socketio_mode == 'gevent':
         # Use gevent mode (higher concurrency but platform limitations)
         try:
@@ -255,7 +289,7 @@ def init_socketio(app, datastore):
         # Invalid mode specified, use default
         async_mode = 'threading'
         logger.warning(f"Invalid SOCKETIO_MODE='{socketio_mode}', using default {async_mode} mode for Socket.IO")
-    
+
     # Log platform info for debugging
     logger.info(f"Platform: {system}, Python: {python_version.major}.{python_version.minor}, Socket.IO mode: {async_mode}")
 
@@ -293,7 +327,6 @@ def init_socketio(app, datastore):
             emit_flash=False
         )
 
-
     @socketio.on('connect')
     def handle_connect():
         """Handle client connection"""
@@ -371,4 +404,4 @@ def init_socketio(app, datastore):
 
     logger.info("Socket.IO initialized and attached to main Flask app")
     logger.info(f"Socket.IO: Registered event handlers: {socketio.handlers if hasattr(socketio, 'handlers') else 'No handlers found'}")
-    return socketio
+    return socketio

changedetectionio/safe_jinja.py

@@ -10,9 +10,15 @@ import os
 
 JINJA2_MAX_RETURN_PAYLOAD_SIZE = 1024 * int(os.getenv("JINJA2_MAX_RETURN_PAYLOAD_SIZE_KB", 1024 * 10))
 
-
+# This is used for notifications etc, so actually it's OK to send custom HTML such as <a href> etc, but it should limit what data is available.
+# (Which also limits available functions that could be called)
 def render(template_str, **args: t.Any) -> str:
     jinja2_env = jinja2.sandbox.ImmutableSandboxedEnvironment(extensions=['jinja2_time.TimeExtension'])
     output = jinja2_env.from_string(template_str).render(args)
     return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]
 
+def render_fully_escaped(content):
+    env = jinja2.sandbox.ImmutableSandboxedEnvironment(autoescape=True)
+    template = env.from_string("{{ some_html|e }}")
+    return template.render(some_html=content)
+

changedetectionio/static/js/plugins.js

@@ -159,6 +159,7 @@
         // Return the current request in case it's needed
         return requests[namespace];
     };
+
 })(jQuery);
 
 

changedetectionio/static/js/realtime.js

@@ -48,13 +48,14 @@ $(document).ready(function () {
             // Connect to Socket.IO on the same host/port, with path from template
             const socket = io({
                 path: socketio_url,  // This will be the path prefix like "/app/socket.io" from the template
-                transports: ['polling', 'websocket'],  // Try WebSocket but fall back to polling
-                reconnectionDelay: 1000,
-                reconnectionAttempts: 15
+                transports: ['websocket', 'polling'],
+                reconnectionDelay: 3000,
+                reconnectionAttempts: 25
             });
 
             // Connection status logging
             socket.on('connect', function () {
+                $('#realtime-conn-error').hide();
                 console.log('Socket.IO connected with path:', socketio_url);
                 console.log('Socket transport:', socket.io.engine.transport.name);
                 bindSocketHandlerButtonsEvents(socket);
@@ -74,7 +75,8 @@ $(document).ready(function () {
 
             socket.on('disconnect', function (reason) {
                 console.log('Socket.IO disconnected, reason:', reason);
-                $('.ajax-op').off('.socketHandlerNamespace')
+                $('.ajax-op').off('.socketHandlerNamespace');
+                $('#realtime-conn-error').show();
             });
 
             socket.on('queue_size', function (data) {
@@ -92,8 +94,28 @@ $(document).ready(function () {
                 }
             });
 
-            // Listen for periodically emitted watch data
-            console.log('Adding watch_update event listener');
+            socket.on('notification_event', function (data) {
+                console.log(`Stub handler for notification_event ${data.watch_uuid}`)
+            });
+
+            socket.on('watch_deleted', function (data) {
+                $('tr[data-watch-uuid="' + data.uuid + '"] td').fadeOut(500, function () {
+                    $(this).closest('tr').remove();
+                });
+            });
+
+            // So that the favicon is only updated when the server has written the scraped favicon to disk.
+            socket.on('watch_bumped_favicon', function (watch) {
+                const $watchRow = $(`tr[data-watch-uuid="${watch.uuid}"]`);
+                if ($watchRow.length) {
+                    $watchRow.addClass('has-favicon');
+                    // Because the event could be emitted from a process that is outside the app context, url_for() might not work.
+                    // Lets use url_for at template generation time to give us a PLACEHOLDER instead
+                    let favicon_url = favicon_baseURL.replace('/PLACEHOLDER', `/${watch.uuid}?cache=${watch.event_timestamp}`);
+                    console.log(`Setting favicon for UUID - ${watch.uuid} - ${favicon_url}`);
+                    $('img.favicon', $watchRow).attr('src', favicon_url);
+                }
+            })
 
             socket.on('watch_update', function (data) {
                 const watch = data.watch;
@@ -104,29 +126,28 @@ $(document).ready(function () {
                 console.log(`${watch.event_timestamp} - Watch update ${watch.uuid} - Checking now - ${watch.checking_now} - UUID in URL ${window.location.href.includes(watch.uuid)}`);
                 console.log('Watch data:', watch);
                 console.log('General stats:', general_stats);
-                
+
                 // Updating watch table rows
                 const $watchRow = $('tr[data-watch-uuid="' + watch.uuid + '"]');
                 console.log('Found watch row elements:', $watchRow.length);
-                
+
                 if ($watchRow.length) {
                     $($watchRow).toggleClass('checking-now', watch.checking_now);
                     $($watchRow).toggleClass('queued', watch.queued);
                     $($watchRow).toggleClass('unviewed', watch.unviewed);
                     $($watchRow).toggleClass('has-error', watch.has_error);
+                    $($watchRow).toggleClass('has-favicon', watch.has_favicon);
                     $($watchRow).toggleClass('notification_muted', watch.notification_muted);
                     $($watchRow).toggleClass('paused', watch.paused);
                     $($watchRow).toggleClass('single-history', watch.history_n === 1);
                     $($watchRow).toggleClass('multiple-history', watch.history_n >= 2);
 
                     $('td.title-col .error-text', $watchRow).html(watch.error_text)
-
                     $('td.last-changed', $watchRow).text(watch.last_changed_text)
-
                     $('td.last-checked .innertext', $watchRow).text(watch.last_checked_text)
                     $('td.last-checked', $watchRow).data('timestamp', watch.last_checked).data('fetchduration', watch.fetch_time);
                     $('td.last-checked', $watchRow).data('eta_complete', watch.last_checked + watch.fetch_time);
-                    
+
                     console.log('Updated UI for watch:', watch.uuid);
                 }
 

changedetectionio/static/js/watch-overview.js

@@ -16,6 +16,12 @@ $(function () {
         $('#op_extradata').val(prompt("Enter a tag name"));
     });
 
+
+    $('.history-link').click(function (e) {
+        // Incase they click 'back' in the browser, it should be removed.
+        $(this).closest('tr').removeClass('unviewed');
+    });
+
     $('.with-share-link > *').click(function () {
         $("#copied-clipboard").remove();
 

changedetectionio/static/styles/package.json

@@ -3,15 +3,16 @@
   "version": "0.0.3",
   "description": "",
   "main": "index.js",
-  "scripts": {
-    "watch": "node-sass -w scss -o .",
-    "build": "node-sass scss -o ."
+  "engines": {
+    "node": ">=18.0.0"
   },
-  "author": "",
-  "license": "ISC",
+  "scripts": {
+    "watch": "sass --watch scss:. --style=compressed --no-source-map",
+    "build": "sass scss:. --style=compressed --no-source-map"
+  },
+  "author": "Leigh Morresi / Web Technologies s.r.o.",
+  "license": "Apache",
   "dependencies": {
-    "node-sass": "^7.0.0",
-    "tar": "^6.1.9",
-    "trim-newlines": "^3.0.1"
+    "sass": "^1.77.8"
   }
 }

changedetectionio/static/styles/scss/diff.scss

@@ -1,4 +1,4 @@
-@import "parts/_variables.scss";
+@use "parts/variables";
 
 #diff-ui {
 

changedetectionio/static/styles/scss/parts/_extra_proxies.scss

@@ -64,17 +64,17 @@ body.proxy-check-active {
 #recommended-proxy {
   display: grid;
   gap: 2rem;
-    @media  (min-width: 991px) {
-      grid-template-columns: repeat(2, 1fr);
-    }
+  padding-bottom: 1em;
+  
+  @media  (min-width: 991px) {
+    grid-template-columns: repeat(2, 1fr);
+  }
 
   > div {
     border: 1px #aaa solid;
     border-radius: 4px;
     padding: 1em;
   }
-
-  padding-bottom: 1em;
 }
 
 #extra-proxies-setting {

changedetectionio/static/styles/scss/parts/_lister_extra.scss

@@ -0,0 +1,92 @@
+.watch-table {
+  &.favicon-not-enabled {
+    tr {
+      .favicon {
+        display: none;
+      }
+    }
+  }
+  &.favicon-enabled {
+    tr {
+      /* make the icons and the text inline-ish */
+      td.inline.title-col {
+        .flex-wrapper {
+          display: flex;
+          align-items: center;
+          gap: 4px;
+        }
+      }
+    }
+  }
+
+  td,
+  th {
+    vertical-align: middle;
+  }
+
+  tr.has-favicon {
+    &.unviewed {
+      img.favicon {
+        opacity: 1.0 !important;
+      }
+    }
+  }
+
+  .status-icons {
+    white-space: nowrap;
+    display: flex;
+    align-items: center; /* Vertical centering */
+    gap: 4px; /* Space between image and text */
+    > * {
+      vertical-align: middle;
+    }
+  }
+}
+
+.title-col {
+  /* Optional, for spacing */
+  padding: 10px;
+}
+
+.title-wrapper {
+  display: flex;
+  align-items: center; /* Vertical centering */
+  gap: 10px; /* Space between image and text */
+}
+
+/* Make sure .title-col-inner doesn't collapse or misalign */
+.title-col-inner {
+  display: inline-block;
+  vertical-align: middle;
+}
+
+/* favicon styling */
+.watch-table {
+  img.favicon {
+    vertical-align: middle;
+    max-width: 25px;
+    max-height: 25px;
+    height: 25px;
+    padding-right: 4px;
+  }
+
+    // Reserved for future use
+  /*  &.thumbnail-type-screenshot {
+      tr.has-favicon {
+        td.inline.title-col {
+          img.thumbnail {
+            background-color: #fff; !* fallback bg for SVGs without bg *!
+            border-radius: 4px; !* subtle rounded corners *!
+            border: 1px solid #ddd; !* light border for contrast *!
+            box-shadow: 0 2px 6px rgba(0, 0, 0, 0.15); !* soft shadow *!
+            filter: contrast(1.05) saturate(1.1) drop-shadow(0 0 0.5px rgba(0, 0, 0, 0.2));
+            object-fit: cover; !* crop/fill if needed *!
+            opacity: 0.8;
+            max-width: 30px;
+            max-height: 30px;
+            height: 30px;
+          }
+        }
+      }
+    }*/
+}

changedetectionio/static/styles/scss/parts/_preview_text_filter.scss

@@ -1,4 +1,4 @@
-@import "minitabs";
+@use "minitabs";
 
 body.preview-text-enabled {
 

changedetectionio/static/styles/scss/parts/_variables.scss

@@ -78,6 +78,7 @@
   --color-text-watch-tag-list: rgba(231, 0, 105, 0.4);
   --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
   --color-background-new-watch-input: var(--color-white);
+  --color-background-new-watch-input-transparent: rgba(255, 255, 255, 0.1);
   --color-text-new-watch-input: var(--color-text);
 
   --color-border-input: var(--color-grey-500);
@@ -112,6 +113,7 @@ html[data-darkmode="true"] {
   --color-background-gradient-third: #4d2c64;
 
   --color-background-new-watch-input: var(--color-grey-100);
+  --color-background-new-watch-input-transparent: var(--color-grey-100);
   --color-text-new-watch-input: var(--color-text);
   --color-background-table-thead: var(--color-grey-200);
   --color-table-background: var(--color-grey-300);

changedetectionio/static/styles/scss/parts/_watch_table-mobile.scss

@@ -0,0 +1,178 @@
+$grid-col-checkbox: 20px;
+$grid-col-watch: 100px;
+$grid-gap: 0.5rem;
+
+
+@media (max-width: 767px) {
+
+  /*
+  Max width before this PARTICULAR table gets nasty
+  This query will take effect for any screen smaller than 760px
+  and also iPads specifically.
+  */
+  .watch-table {
+    /* make headings work on mobile */
+    thead {
+      display: block;
+
+      tr {
+        th {
+          display: inline-block;
+          // Hide the "Last" text for smaller screens
+          @media (max-width: 768px) {
+            .hide-on-mobile {
+              display: none;
+            }
+          }
+        }
+      }
+
+      .empty-cell {
+        display: none;
+      }
+    }
+
+
+    .last-checked {
+      margin-left: calc($grid-col-checkbox + $grid-gap);
+
+      > span {
+        vertical-align: middle;
+      }
+    }
+
+    .last-changed {
+      margin-left: calc($grid-col-checkbox + $grid-gap);
+    }
+
+    .last-checked::before {
+      color: var(--color-text);
+      content: "Last Checked ";
+    }
+
+    .last-changed::before {
+      color: var(--color-text);
+      content: "Last Changed ";
+    }
+
+    /* Force table to not be like tables anymore */
+    td.inline {
+      display: inline-block;
+    }
+
+    .pure-table td,
+    .pure-table th {
+      border: none;
+    }
+
+    td {
+      /* Behave  like a "row" */
+      border: none;
+      border-bottom: 1px solid var(--color-border-watch-table-cell);
+      vertical-align: middle;
+
+      &:before {
+        /* Top/left values mimic padding */
+        top: 6px;
+        left: 6px;
+        width: 45%;
+        padding-right: 10px;
+        white-space: nowrap;
+      }
+    }
+
+    &.pure-table-striped {
+      tr {
+        background-color: var(--color-table-background);
+      }
+
+      tr:nth-child(2n-1) {
+        background-color: var(--color-table-stripe);
+      }
+
+      tr:nth-child(2n-1) td {
+        background-color: inherit;
+      }
+    }
+  }
+}
+
+@media (max-width: 767px) {
+  .watch-table {
+    tbody {
+      tr {
+        padding-bottom: 10px;
+        padding-top: 10px;
+        display: grid;
+        grid-template-columns: $grid-col-checkbox 1fr $grid-col-watch;
+        grid-template-rows: auto auto auto auto;
+        gap: $grid-gap;
+
+        .counter-i {
+          display: none;
+        }
+
+        td.checkbox-uuid {
+          display: grid;
+          place-items: center;
+        }
+
+        td.inline {
+          /* display: block !important;;*/
+        }
+
+        > td {
+          border-bottom: none;
+        }
+
+        > td.title-col {
+          grid-column: 1 / -1;
+          grid-row: 1;
+          .watch-title {
+            font-size: 0.92rem;
+          }
+          .link-spread {
+            display: none;
+          }
+        }
+
+        > td.last-checked {
+          grid-column: 1 / -1;
+          grid-row: 2;
+        }
+
+        > td.last-changed {
+          grid-column: 1 / -1;
+          grid-row: 3;
+        }
+
+        > td.checkbox-uuid {
+          grid-column: 1;
+          grid-row: 4;
+        }
+
+        > td.buttons {
+          grid-column: 2;
+          grid-row: 4;
+          display: flex;
+          align-items: center;
+          justify-content: flex-start;
+        }
+
+        > td.watch-controls {
+          grid-column: 3;
+          grid-row: 4;
+          display: grid;
+          place-items: center;
+
+          a img {
+            padding: 10px;
+          }
+        }
+      }
+    }
+  }
+  .pure-table td {
+    padding: 3px !important;
+  }
+}

changedetectionio/static/styles/scss/parts/_watch_table.scss

@@ -7,6 +7,7 @@
     &.unviewed {
       font-weight: bold;
     }
+
     color: var(--color-watch-table-row-text);
   }
 
@@ -17,11 +18,13 @@
     &.title-col {
       word-break: break-all;
       white-space: normal;
-      a::after {
-        content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
-        margin: 0 3px 0 5px;
-      }
     }
+
+    a.external::after {
+      content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
+      margin: 0 3px 0 5px;
+    }
+
   }
 
 
@@ -46,17 +49,17 @@
   /* Row with 'checking-now' */
   tr.checking-now {
     td:first-child {
-        position: relative;
+      position: relative;
     }
 
     td:first-child::before {
-        content: "";
-        position: absolute;
-        top: 0;
-        bottom: 0;
-        left: 0;
-        width: 3px;
-        background-color: #293eff;
+      content: "";
+      position: absolute;
+      top: 0;
+      bottom: 0;
+      left: 0;
+      width: 3px;
+      background-color: #293eff;
     }
 
     td.last-checked {
@@ -107,6 +110,7 @@
 
   tr.has-error {
     color: var(--color-watch-table-error);
+
     .error-text {
       display: block !important;
     }
@@ -117,6 +121,7 @@
       display: inline-block !important;
     }
   }
+
   tr.multiple-history {
     a.history-link {
       display: inline-block !important;
@@ -124,5 +129,3 @@
   }
 }
 
-
-

changedetectionio/static/styles/scss/styles.scss

@@ -2,21 +2,25 @@
  * -- BASE STYLES --
  */
 
-@import "parts/_arrows";
-@import "parts/_browser-steps";
-@import "parts/_extra_proxies";
-@import "parts/_extra_browsers";
-@import "parts/_pagination";
-@import "parts/_spinners";
-@import "parts/_variables";
-@import "parts/_darkmode";
-@import "parts/_menu";
-@import "parts/_love";
-@import "parts/preview_text_filter";
-@import "parts/_watch_table";
-@import "parts/_edit";
-@import "parts/_conditions_table";
-@import "parts/_socket";
+@use "parts/variables";
+@use "parts/arrows";
+@use "parts/browser-steps";
+@use "parts/extra_proxies";
+@use "parts/extra_browsers";
+@use "parts/pagination";
+@use "parts/spinners";
+@use "parts/darkmode";
+@use "parts/menu";
+@use "parts/love";
+@use "parts/preview_text_filter";
+@use "parts/watch_table";
+@use "parts/watch_table-mobile";
+@use "parts/edit";
+@use "parts/conditions_table";
+@use "parts/lister_extra";
+@use "parts/socket";
+@use "parts/visualselector";
+
 
 body {
   color: var(--color-text);
@@ -184,8 +188,15 @@ code {
   @extend .inline-tag;
 }
 
+@media (min-width: 768px) {
+  .box {
+    margin: 0 1em !important;
+  }
+}
+
 .box {
-  max-width: 80%;
+  max-width: 100%;
+  margin: 0 0.3em;
   flex-direction: column;
   display: flex;
   justify-content: center;
@@ -279,7 +290,7 @@ a.pure-button-selected {
   font-size: 65%;
   border-bottom-left-radius: initial;
   border-bottom-right-radius: initial;
-
+  margin-right: 4px;
   &.active {
     background: var(--color-background-button-tag-active);
     font-weight: bold;
@@ -372,11 +383,32 @@ label {
   }
 }
 
+// Some field colouring for transperant field
+.pure-form input[type=text].transparent-field {
+  background-color:  var(--color-background-new-watch-input-transparent) !important;
+  color: var(--color-white) !important;
+  border: 1px solid rgba(255, 255, 255, 0.2) !important;
+  box-shadow: none !important;
+  -webkit-box-shadow: none !important;
+  &::placeholder {
+    opacity: 0.5;
+    color: rgba(255, 255, 255, 0.7);
+    font-weight: lighter;
+  }
+}
+
 #new-watch-form {
   background: var(--color-background-new-watch-form);
   padding: 1em;
   border-radius: 10px;
   margin-bottom: 1em;
+  max-width: 100%;
+
+  #url {
+    &::placeholder {
+      font-weight: bold;
+    }
+  }
 
   input {
     display: inline-block;
@@ -397,12 +429,13 @@ label {
     font-weight: bold;
   }
 
-  #watch-add-wrapper-zone {
 
+  #watch-add-wrapper-zone {
     @media only screen and (min-width: 760px) {
       display: flex;
       gap: 0.3rem;
       flex-direction: row;
+      min-width: 70vw;
     }
     /* URL field grows always, other stay static in width */
     > span {
@@ -424,6 +457,22 @@ label {
       }
     }
   }
+
+  #watch-group-tag {
+    font-size: 0.9rem;
+    padding: 0.3rem;
+    display: flex;
+    align-items: center;
+    gap: 0.5rem;
+    color: var(--color-white);
+    label, input {
+      margin: 0;
+    }
+
+    input {
+      flex: 1;
+    }
+  }
 }
 
 
@@ -620,10 +669,6 @@ footer {
 
 @media only screen and (max-width: 760px),
 (min-device-width: 768px) and (max-device-width: 1024px) {
-  .box {
-    max-width: 95%
-  }
-
   .edit-form {
     padding: 0.5em;
     margin: 0;
@@ -659,114 +704,6 @@ footer {
     width: 100%;
   }
 
-  /*
-  Max width before this PARTICULAR table gets nasty
-  This query will take effect for any screen smaller than 760px
-  and also iPads specifically.
-  */
-  .watch-table {
-    /* make headings work on mobile */
-    thead {
-      display: block;
-      tr {
-        th {
-          display: inline-block;
-          // Hide the "Last" text for smaller screens
-          @media (max-width: 768px) {
-            .hide-on-mobile {
-              display: none; 
-            }
-          }
-        }
-      }
-      .empty-cell {
-        display: none;
-      }
-    }
-
-    /* Force table to not be like tables anymore */
-    tbody {
-      td,
-      tr {
-        display: block;
-      }
-    }
-
-    tbody {
-      tr {
-        display: flex;
-        flex-wrap: wrap;
-
-        // The third child of each row will take up the remaining space
-        // This is useful for the URL column, which should expand to fill the remaining space
-        :nth-child(3) {
-          flex-grow: 1;
-        }
-        // The last three children (from the end) of each row will take up the full width
-        // This is useful for the "Last Checked", "Last Changed", and the action buttons columns, which should each take up the full width
-        :nth-last-child(-n+3) {
-          flex-basis: 100%;
-        }
-      }
-    }
-
-    .last-checked {
-      >span {
-        vertical-align: middle;
-      }
-    }
-
-    .last-checked::before {
-      color: var(--color-last-checked);
-      content: "Last Checked ";
-    }
-
-    .last-changed::before {
-      color: var(--color-last-checked);
-      content: "Last Changed ";
-    }
-
-    /* Force table to not be like tables anymore */
-    td.inline {
-      display: inline-block;
-    }
-
-    .pure-table td,
-    .pure-table th {
-      border: none;
-    }
-
-    td {
-      /* Behave  like a "row" */
-      border: none;
-      border-bottom: 1px solid var(--color-border-watch-table-cell);
-      vertical-align: middle;
-
-      &:before {
-        /* Top/left values mimic padding */
-        top: 6px;
-        left: 6px;
-        width: 45%;
-        padding-right: 10px;
-        white-space: nowrap;
-      }
-    }
-
-    &.pure-table-striped {
-      tr {
-        background-color: var(--color-table-background);
-      }
-
-      tr:nth-child(2n-1) {
-        background-color: var(--color-table-stripe);
-      }
-
-      tr:nth-child(2n-1) td {
-        background-color: inherit;
-      }
-    }
-
-  }
 }
 
 .pure-table {
@@ -1021,8 +958,6 @@ ul {
   }
 }
 
-@import "parts/_visualselector";
-
 #webdriver_delay {
     width: 5em;
 }
@@ -1140,19 +1075,23 @@ ul {
 
 
 #quick-watch-processor-type {
-  color: #fff;
-  ul {
-    padding: 0.3rem;
-
+  ul#processor {
+    color: #fff;
+    padding-left: 0px;
     li {
       list-style: none;
-      font-size: 0.8rem;
-      > * {
-        display: inline-block;
-      }
+      font-size: 0.9rem;
+      display: grid;
+      grid-template-columns: auto 1fr;
+      align-items: center;
+      gap: 0.5rem;
+      margin-bottom: 0.5rem;
     }
   }
-
+  label, input {
+    padding: 0;
+    margin: 0;
+  }
 }
 
 .restock-label {
@@ -1190,3 +1129,12 @@ ul {
   vertical-align: middle;
 }
 
+#realtime-conn-error {
+  position: absolute;
+  bottom: 0;
+  left: 30px;
+  background: var(--color-warning);
+  padding: 10px;
+  font-size: 0.8rem;
+  color: #fff;
+}

changedetectionio/store.py

@@ -13,6 +13,7 @@ import json
 import os
 import re
 import secrets
+import sys
 import threading
 import time
 import uuid as uuid_builder
@@ -43,9 +44,11 @@ class ChangeDetectionStore:
     def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
         # Should only be active for docker
         # logging.basicConfig(filename='/dev/stdout', level=logging.INFO)
+        from deepmerge import always_merger
+
         self.__data = App.model()
         self.datastore_path = datastore_path
-        self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
+        self.json_store_path = os.path.join(self.datastore_path, "url-watches.json")
         logger.info(f"Datastore path is '{self.json_store_path}'")
         self.needs_write = False
         self.start_time = time.time()
@@ -74,14 +77,8 @@ class ChangeDetectionStore:
                     self.__data['app_guid'] = from_disk['app_guid']
 
                 if 'settings' in from_disk:
-                    if 'headers' in from_disk['settings']:
-                        self.__data['settings']['headers'].update(from_disk['settings']['headers'])
-
-                    if 'requests' in from_disk['settings']:
-                        self.__data['settings']['requests'].update(from_disk['settings']['requests'])
-
-                    if 'application' in from_disk['settings']:
-                        self.__data['settings']['application'].update(from_disk['settings']['application'])
+                    # update the modal with whats on disk
+                    self.__data['settings'] = always_merger.merge(from_disk['settings'], self.__data['settings'])
 
                 # Convert each existing watch back to the Watch.model object
                 for uuid, watch in self.__data['watching'].items():
@@ -118,14 +115,12 @@ class ChangeDetectionStore:
         test_list = self.proxy_list
 
         # Helper to remove password protection
-        password_reset_lockfile = "{}/removepassword.lock".format(self.datastore_path)
+        password_reset_lockfile = os.path.join(self.datastore_path, "removepassword.lock")
         if path.isfile(password_reset_lockfile):
             self.__data['settings']['application']['password'] = False
             unlink(password_reset_lockfile)
 
         if not 'app_guid' in self.__data:
-            import os
-            import sys
             if "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ:
                 self.__data['app_guid'] = "test-" + str(uuid_builder.uuid4())
             else:
@@ -386,9 +381,9 @@ class ChangeDetectionStore:
         return new_uuid
 
     def visualselector_data_is_ready(self, watch_uuid):
-        output_path = "{}/{}".format(self.datastore_path, watch_uuid)
-        screenshot_filename = "{}/last-screenshot.png".format(output_path)
-        elements_index_filename = "{}/elements.deflate".format(output_path)
+        output_path = os.path.join(self.datastore_path, watch_uuid)
+        screenshot_filename = os.path.join(output_path, "last-screenshot.png")
+        elements_index_filename = os.path.join(output_path, "elements.deflate")
         if path.isfile(screenshot_filename) and  path.isfile(elements_index_filename) :
             return True
 
@@ -412,11 +407,7 @@ class ChangeDetectionStore:
                 # system was out of memory, out of RAM etc
                 with open(self.json_store_path+".tmp", 'w') as json_file:
                     # Use compact JSON in production for better performance
-                    debug_mode = os.environ.get('CHANGEDETECTION_DEBUG', 'false').lower() == 'true'
-                    if debug_mode:
-                        json.dump(data, json_file, indent=4)
-                    else:
-                        json.dump(data, json_file, separators=(',', ':'))
+                    json.dump(data, json_file, indent=2)
                 os.replace(self.json_store_path+".tmp", self.json_store_path)
             except Exception as e:
                 logger.error(f"Error writing JSON!! (Main JSON file save was skipped) : {str(e)}")
@@ -478,7 +469,7 @@ class ChangeDetectionStore:
 
         # Load from external config file
         if path.isfile(proxy_list_file):
-            with open("{}/proxies.json".format(self.datastore_path)) as f:
+            with open(os.path.join(self.datastore_path, "proxies.json")) as f:
                 proxy_list = json.load(f)
 
         # Mapping from UI config if available
@@ -736,10 +727,10 @@ class ChangeDetectionStore:
                 logger.critical(f"Applying update_{update_n}")
                 # Wont exist on fresh installs
                 if os.path.exists(self.json_store_path):
-                    shutil.copyfile(self.json_store_path, self.datastore_path+"/url-watches-before-{}.json".format(update_n))
+                    shutil.copyfile(self.json_store_path, os.path.join(self.datastore_path, f"url-watches-before-{update_n}.json"))
 
                 try:
-                    update_method = getattr(self, "update_{}".format(update_n))()
+                    update_method = getattr(self, f"update_{update_n}")()
                 except Exception as e:
                     logger.error(f"Error while trying update_{update_n}")
                     logger.error(e)

changedetectionio/templates/_common_fields.html

@@ -74,7 +74,7 @@
                                     </tr>
                                     <tr>
                                         <td><code>{{ '{{watch_tag}}' }}</code></td>
-                                        <td>The watch label / tag</td>
+                                        <td>The watch group / tag</td>
                                     </tr>
                                     <tr>
                                         <td><code>{{ '{{preview_url}}' }}</code></td>

changedetectionio/templates/base.html

@@ -236,6 +236,7 @@
     <script src="{{url_for('static_content', group='js', filename='toggle-theme.js')}}" defer></script>
 
     <div id="checking-now-fixed-tab" style="display: none;"><span class="spinner"></span><span>&nbsp;Checking now</span></div>
+    <div id="realtime-conn-error" style="display:none">Offline</div>
   </body>
 
 </html>

changedetectionio/tests/fetchers/test_content.py

@@ -2,19 +2,24 @@
 
 import time
 from flask import url_for
+import os
 from ..util import live_server_setup, wait_for_all_checks
 import logging
 
+
 # Requires playwright to be installed
 def test_fetch_webdriver_content(client, live_server, measure_memory_usage):
-   #  live_server_setup(live_server) # Setup on conftest per function
+    #  live_server_setup(live_server) # Setup on conftest per function
 
     #####################
     res = client.post(
         url_for("settings.settings_page"),
-        data={"application-empty_pages_are_a_change": "",
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_webdriver"},
+        data={
+            "application-empty_pages_are_a_change": "",
+            "requests-time_between_check-minutes": 180,
+            'application-fetch_backend': "html_webdriver",
+            'application-ui-favicons_enabled': "y",
+        },
         follow_redirects=True
     )
 
@@ -30,11 +35,51 @@ def test_fetch_webdriver_content(client, live_server, measure_memory_usage):
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
 
-
     res = client.get(
         url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
     )
     logging.getLogger().info("Looking for correct fetched HTML (text) from server")
-
     assert b'cool it works' in res.data
+
+    # Favicon scraper check, favicon only so far is fetched when in browser mode (not requests mode)
+    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
+        uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+        res = client.get(
+            url_for("watchlist.index"),
+        )
+        # The UI can access it here
+        assert f'src="/static/favicon/{uuid}'.encode('utf8') in res.data
+
+        # Attempt to fetch it, make sure that works
+        res = client.get(url_for('static_content', group='favicon', filename=uuid))
+        assert res.status_code == 200
+        assert len(res.data) > 10
+
+        # Check the API also returns it
+        api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+        res = client.get(
+            url_for("watchfavicon", uuid=uuid),
+            headers={'x-api-key': api_key}
+        )
+        assert res.status_code == 200
+        assert len(res.data) > 10
+
+    ##################### disable favicons check
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={
+            "requests-time_between_check-minutes": 180,
+            'application-ui-favicons_enabled': "",
+            "application-empty_pages_are_a_change": "",
+        },
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    res = client.get(
+        url_for("watchlist.index"),
+    )
+    # The UI can access it here
+    assert f'src="/static/favicon'.encode('utf8') not in res.data

changedetectionio/tests/realtime/test_socketio.py

@@ -1,72 +0,0 @@
-import asyncio
-import socketio
-from aiohttp import web
-
-SOCKETIO_URL = 'ws://localhost.localdomain:5005'
-SOCKETIO_PATH = "/socket.io"
-NUM_CLIENTS = 1
-
-clients = []
-shutdown_event = asyncio.Event()
-
-class WatchClient:
-    def __init__(self, client_id: int):
-        self.client_id = client_id
-        self.i_got_watch_update_event = False
-        self.sio = socketio.AsyncClient(reconnection_attempts=50, reconnection_delay=1)
-
-        @self.sio.event
-        async def connect():
-            print(f"[Client {self.client_id}] Connected")
-
-        @self.sio.event
-        async def disconnect():
-            print(f"[Client {self.client_id}] Disconnected")
-
-        @self.sio.on("watch_update")
-        async def on_watch_update(watch):
-            self.i_got_watch_update_event = True
-            print(f"[Client {self.client_id}] Received update: {watch}")
-
-    async def run(self):
-        try:
-            await self.sio.connect(SOCKETIO_URL, socketio_path=SOCKETIO_PATH, transports=["websocket", "polling"])
-            await self.sio.wait()
-        except Exception as e:
-            print(f"[Client {self.client_id}] Connection error: {e}")
-
-async def handle_check(request):
-    all_received = all(c.i_got_watch_update_event for c in clients)
-    result = "yes" if all_received else "no"
-    print(f"Received HTTP check — returning '{result}'")
-    shutdown_event.set()  # Signal shutdown
-    return web.Response(text=result)
-
-async def start_http_server():
-    app = web.Application()
-    app.add_routes([web.get('/did_all_clients_get_watch_update', handle_check)])
-    runner = web.AppRunner(app)
-    await runner.setup()
-    site = web.TCPSite(runner, '0.0.0.0', 6666)
-    await site.start()
-
-async def main():
-    #await start_http_server()
-
-    for i in range(NUM_CLIENTS):
-        client = WatchClient(i)
-        clients.append(client)
-        asyncio.create_task(client.run())
-
-    await shutdown_event.wait()
-
-    print("Shutting down...")
-    # Graceful disconnect
-    for c in clients:
-        await c.sio.disconnect()
-
-if __name__ == "__main__":
-    try:
-        asyncio.run(main())
-    except KeyboardInterrupt:
-        print("Interrupted")

changedetectionio/tests/test_group.py

@@ -236,39 +236,41 @@ def test_group_tag_notification(client, live_server, measure_memory_usage):
     assert b'Deleted' in res.data
 
 def test_limit_tag_ui(client, live_server, measure_memory_usage):
-    
 
-    test_url = url_for('test_endpoint', _external=True)
-    urls=[]
+    test_url = url_for('test_random_content_endpoint', _external=True)
 
-    for i in range(20):
-        urls.append(test_url+"?x="+str(i)+" test-tag")
-
-    for i in range(20):
-        urls.append(test_url+"?non-grouped="+str(i))
-
-    res = client.post(
+    # A space can label the tag, only the first one will have a tag
+    client.post(
         url_for("imports.import_page"),
-        data={"urls": "\r\n".join(urls)},
+        data={"urls": f"{test_url} test-tag\r\n{test_url}"},
         follow_redirects=True
     )
-
-    assert b"40 Imported" in res.data
+    tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
+    assert tag_uuid
 
     res = client.get(url_for("watchlist.index"))
     assert b'test-tag' in res.data
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
 
-    # All should be here
-    assert res.data.count(b'processor-text_json_diff') == 40
+    # Should be both unviewed
+    res = client.get(url_for("watchlist.index"))
+    assert res.data.count(b' unviewed ') == 2
 
-    tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
 
-    res = client.get(url_for("watchlist.index", tag=tag_uuid))
+    # Now we recheck only the tag
+    client.get(url_for('ui.mark_all_viewed', tag=tag_uuid), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    with open('/tmp/fuck.html', 'wb') as f:
+        f.write(res.data)
+    # Should be only 1 unviewed
+    res = client.get(url_for("watchlist.index"))
+    assert res.data.count(b' unviewed ') == 1
+
 
-    # Just a subset should be here
-    assert b'test-tag' in res.data
-    assert res.data.count(b'processor-text_json_diff') == 20
-    assert b"object at" not in res.data
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)

changedetectionio/tests/test_history_consistency.py

@@ -79,3 +79,48 @@ def test_consistent_history(client, live_server, measure_memory_usage):
     json_db_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, 'url-watches.json')
     with open(json_db_file, 'r') as f:
         assert '"default"' not in f.read(), "'default' probably shouldnt be here, it came from when the 'default' Watch vars were accidently being saved"
+
+
+def test_check_text_history_view(client, live_server):
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("<html>test-one</html>")
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("imports.import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Give the thread time to pick it up
+    wait_for_all_checks(client)
+
+    # Set second version, Make a change
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("<html>test-two</html>")
+
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    res = client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
+    assert b'test-one' in res.data
+    assert b'test-two' in res.data
+
+    # Set third version, Make a change
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("<html>test-three</html>")
+
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    # It should remember the last viewed time, so the first difference is not shown
+    res = client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
+    assert b'test-three' in res.data
+    assert b'test-two' in res.data
+    assert b'test-one' not in res.data
+
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -419,13 +419,20 @@ def check_json_ext_filter(json_filter, client, live_server):
     res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
 
-    res = client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
+    res = client.get(url_for("ui.ui_views.preview_page", uuid="first"))
 
     # We should never see 'ForSale' because we are selecting on 'Sold' in the rule,
     # But we should know it triggered ('unviewed' assert above)
     assert b'ForSale' not in res.data
     assert b'Sold' in res.data
 
+
+    # And the difference should have both?
+
+    res = client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
+    assert b'ForSale' in res.data
+    assert b'Sold' in res.data
+
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 

changedetectionio/tests/test_security.py

@@ -5,8 +5,22 @@ from .util import live_server_setup, wait_for_all_checks
 from .. import strtobool
 
 
-# def test_setup(client, live_server, measure_memory_usage):
-   #  live_server_setup(live_server) # Setup on conftest per function
+def set_original_response():
+    test_return_data = """<html>
+    <head><title>head title</title></head>
+    <body>
+     Some initial text<br>
+     <p>Which is across multiple lines</p>
+     <br>
+     So let's see what happens.  <br>
+     <span class="foobar-detection" style='display:none'></span>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
 
 def test_bad_access(client, live_server, measure_memory_usage):
     
@@ -118,3 +132,33 @@ def test_xss(client, live_server, measure_memory_usage):
     assert b"<img src=x onerror=alert(" not in res.data
     assert b"&lt;img" in res.data
 
+
+def test_xss_watch_last_error(client, live_server, measure_memory_usage):
+    set_original_response()
+    # Add our URL to the import page
+    res = client.post(
+        url_for("imports.import_page"),
+        data={"urls": url_for('test_endpoint', _external=True)},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    wait_for_all_checks(client)
+    res = client.post(
+        url_for("ui.ui_edit.edit_page", uuid="first"),
+        data={
+            "include_filters": '<a href="https://foobar"></a><script>alert(123);</script>',
+            "url": url_for('test_endpoint', _external=True),
+            'fetch_backend': "html_requests"
+        },
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+    wait_for_all_checks(client)
+    res = client.get(url_for("watchlist.index"))
+
+    assert b"<script>alert(123);</script>" not in res.data  # this text should be there
+    assert b'&lt;a href=&#34;https://foobar&#34;&gt;&lt;/a&gt;&lt;script&gt;alert(123);&lt;/script&gt;' in res.data
+    assert b"https://foobar" in res.data # this text should be there
+

changedetectionio/tests/unit/test_jinja2_security.py

@@ -51,6 +51,9 @@ class TestJinja2SSTI(unittest.TestCase):
         for attempt in attempt_list:
             self.assertEqual(len(safe_jinja.render(attempt)), 0, f"string test '{attempt}' is correctly empty")
 
+    def test_jinja2_escaped_html(self):
+        x = safe_jinja.render_fully_escaped('woo <a href="https://google.com">dfdfd</a>')
+        self.assertEqual(x, "woo &lt;a href=&#34;https://google.com&#34;&gt;dfdfd&lt;/a&gt;")
 
 
 if __name__ == '__main__':

docker-compose.yml

@@ -69,6 +69,16 @@ services:
   #        Maximum height of screenshots, default is 16000 px, screenshots will be clipped to this if exceeded.
   #        RAM usage will be higher if you increase this.
   #      - SCREENSHOT_MAX_HEIGHT=16000
+  #
+  #        HTTPS SSL Mode for webserver, unset both of these, you may need to volume mount these files also.
+  #        ./cert.pem:/app/cert.pem and ./privkey.pem:/app/privkey.pem
+  #      - SSL_CERT_FILE=cert.pem
+  #      - SSL_PRIVKEY_FILE=privkey.pem
+  #
+  #        LISTEN_HOST / "host", Same as -h
+  #      - LISTEN_HOST=::
+  #      - LISTEN_HOST=0.0.0.0
+
   
       # Comment out ports: when using behind a reverse proxy , enable networks: etc.
       ports:

requirements.txt

@@ -78,6 +78,7 @@ jq~=1.3; python_version >= "3.8" and sys_platform == "linux"
 # playwright is installed at Dockerfile build time because it's not available on all platforms
 
 pyppeteer-ng==2.0.0rc10
+deepmerge
 
 pyppeteerstealth>=0.0.4
 
@@ -117,6 +118,9 @@ price-parser
 
 # flask_socket_io - incorrect package name, already have flask-socketio above
 
+# So far for detecting correct favicon type, but for other things in the future
+python-magic
+
 # Scheduler - Windows seemed to miss a lot of default timezone info (even "UTC" !)
 tzdata