Adding page title+page link to API

.dockerignore

@@ -33,7 +33,6 @@ venv/
 # Test and development files
 test-datastore/
 tests/
-docs/
 *.md
 !README.md
 

.github/workflows/codeql-analysis.yml

@@ -30,7 +30,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/containers.yml

@@ -39,9 +39,9 @@ jobs:
     # Or if we are in a tagged release scenario.
     if: ${{ github.event.workflow_run.conclusion == 'success' }} || ${{ github.event.release.tag_name }} != ''
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Set up Python 3.11
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: 3.11
 

.github/workflows/pypi-release.yml

@@ -7,9 +7,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
     - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: "3.11"
     - name: Install pypa/build
@@ -34,12 +34,12 @@ jobs:
     - build
     steps:
     - name: Download all the dists
-      uses: actions/download-artifact@v4
+      uses: actions/download-artifact@v5
       with:
         name: python-package-distributions
         path: dist/
     - name: Set up Python 3.11
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: '3.11'
     - name: Test that the basic pip built package runs without error
@@ -72,7 +72,7 @@ jobs:
 
     steps:
     - name: Download all the dists
-      uses: actions/download-artifact@v4
+      uses: actions/download-artifact@v5
       with:
         name: python-package-distributions
         path: dist/

.github/workflows/test-container-build.yml

@@ -46,9 +46,9 @@ jobs:
           - platform: linux/arm64
             dockerfile: ./.github/test/Dockerfile-alpine
     steps:
-        - uses: actions/checkout@v4
+        - uses: actions/checkout@v5
         - name: Set up Python 3.11
-          uses: actions/setup-python@v5
+          uses: actions/setup-python@v6
           with:
             python-version: 3.11
 

.github/workflows/test-only.yml

@@ -7,7 +7,7 @@ jobs:
   lint-code:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Lint with Ruff
         run: |
           pip install ruff
@@ -15,6 +15,10 @@ jobs:
           ruff check . --select E9,F63,F7,F82
           # Complete check with errors treated as warnings
           ruff check . --exit-zero
+      - name: Validate OpenAPI spec
+        run: |
+          pip install openapi-spec-validator
+          python3 -c "from openapi_spec_validator import validate_spec; import yaml; validate_spec(yaml.safe_load(open('docs/api-spec.yaml')))"
 
   test-application-3-10:
     needs: lint-code

.github/workflows/test-stack-reusable-workflow.yml

@@ -20,11 +20,11 @@ jobs:
     env:
       PYTHON_VERSION: ${{ inputs.python-version }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       # Mainly just for link/flake8
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 

Dockerfile

@@ -5,7 +5,6 @@ ARG PYTHON_VERSION=3.11
 FROM python:${PYTHON_VERSION}-slim-bookworm AS builder
 
 # See `cryptography` pin comment in requirements.txt
-ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     g++ \
@@ -17,6 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libxslt-dev \
     make \
     patch \
+    pkg-config \
     zlib1g-dev
 
 RUN mkdir /install
@@ -26,6 +26,14 @@ COPY requirements.txt /requirements.txt
 
 # Use cache mounts and multiple wheel sources for faster ARM builds
 ENV PIP_CACHE_DIR=/tmp/pip-cache
+# Help Rust find OpenSSL for cryptography package compilation on ARM
+ENV PKG_CONFIG_PATH="/usr/lib/pkgconfig:/usr/lib/arm-linux-gnueabihf/pkgconfig:/usr/lib/aarch64-linux-gnu/pkgconfig"
+ENV PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1
+ENV OPENSSL_DIR="/usr"
+ENV OPENSSL_LIB_DIR="/usr/lib/arm-linux-gnueabihf"
+ENV OPENSSL_INCLUDE_DIR="/usr/include/openssl"
+# Additional environment variables for cryptography Rust build
+ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
 RUN --mount=type=cache,target=/tmp/pip-cache \
     pip install \
     --extra-index-url https://www.piwheels.org/simple \
@@ -76,6 +84,11 @@ EXPOSE 5000
 
 # The actual flask app module
 COPY changedetectionio /app/changedetectionio
+
+# Also for OpenAPI validation wrapper - needs the YML
+RUN [ ! -d "/app/docs" ] && mkdir /app/docs
+COPY docs/api-spec.yaml /app/docs/api-spec.yaml
+
 # Starting wrapper
 COPY changedetection.py /app/changedetection.py
 

MANIFEST.in

@@ -1,7 +1,7 @@
 recursive-include changedetectionio/api *
 recursive-include changedetectionio/blueprint *
-recursive-include changedetectionio/content_fetchers *
 recursive-include changedetectionio/conditions *
+recursive-include changedetectionio/content_fetchers *
 recursive-include changedetectionio/model *
 recursive-include changedetectionio/notification *
 recursive-include changedetectionio/processors *
@@ -9,6 +9,7 @@ recursive-include changedetectionio/realtime *
 recursive-include changedetectionio/static *
 recursive-include changedetectionio/templates *
 recursive-include changedetectionio/tests *
+recursive-include changedetectionio/widgets *
 prune changedetectionio/static/package-lock.json
 prune changedetectionio/static/styles/node_modules
 prune changedetectionio/static/styles/package-lock.json

README.md

@@ -280,7 +280,10 @@ Excel import is recommended - that way you can better organise tags/groups of we
 
 ## API Support
 
-Supports managing the website watch list [via our API](https://changedetection.io/docs/api_v1/index.html)
+Full REST API for programmatic management of watches, tags, notifications and more. 
+
+- **[Interactive API Documentation](https://changedetection.io/docs/api_v1/index.html)** - Complete API reference with live testing
+- **[OpenAPI Specification](docs/api-spec.yaml)** - Generate SDKs for any programming language
 
 ## Support us
 

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.50.8'
+__version__ = '0.50.13'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
@@ -35,13 +35,22 @@ def sigshutdown_handler(_signo, _stack_frame):
     app.config.exit.set()
     datastore.stop_thread = True
     
-    # Shutdown workers immediately
+    # Shutdown workers and queues immediately
     try:
         from changedetectionio import worker_handler
         worker_handler.shutdown_workers()
     except Exception as e:
         logger.error(f"Error shutting down workers: {str(e)}")
     
+    # Close janus queues properly
+    try:
+        from changedetectionio.flask_app import update_q, notification_q
+        update_q.close()
+        notification_q.close()
+        logger.debug("Janus queues closed successfully")
+    except Exception as e:
+        logger.critical(f"CRITICAL: Failed to close janus queues: {e}")
+    
     # Shutdown socketio server fast
     from changedetectionio.flask_app import socketio_server
     if socketio_server and hasattr(socketio_server, 'shutdown'):

changedetectionio/api/Import.py

@@ -3,7 +3,7 @@ from changedetectionio.strtobool import strtobool
 from flask_restful import abort, Resource
 from flask import request
 import validators
-from . import auth
+from . import auth, validate_openapi_request
 
 
 class Import(Resource):
@@ -12,17 +12,9 @@ class Import(Resource):
         self.datastore = kwargs['datastore']
 
     @auth.check_token
+    @validate_openapi_request('importWatches')
     def post(self):
-        """
+        """Import a list of watched URLs."""
-        @api {post} /api/v1/import Import a list of watched URLs
-        @apiDescription Accepts a line-feed separated list of URLs to import, additionally with ?tag_uuids=(tag  id), ?tag=(name), ?proxy={key}, ?dedupe=true (default true) one URL per line.
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/import --data-binary @list-of-sites.txt -H"x-api-key:8a111a21bc2f8f1dd9b9353bbd46049a"
-        @apiName Import
-        @apiGroup Watch
-        @apiSuccess (200) {List} OK List of watch UUIDs added
-        @apiSuccess (500) {String} ERR Some other error
-        """
 
         extras = {}
 

changedetectionio/api/Notifications.py

@@ -1,9 +1,7 @@
 from flask_expects_json import expects_json
-from flask_restful import Resource
+from flask_restful import Resource, abort
-from . import auth
-from flask_restful import abort, Resource
 from flask import request
-from . import auth
+from . import auth, validate_openapi_request
 from . import schema_create_notification_urls, schema_delete_notification_urls
 
 class Notifications(Resource):
@@ -12,19 +10,9 @@ class Notifications(Resource):
         self.datastore = kwargs['datastore']
 
     @auth.check_token
+    @validate_openapi_request('getNotifications')
     def get(self):
-        """
+        """Return Notification URL List."""
-        @api {get} /api/v1/notifications Return Notification URL List
-        @apiDescription Return the Notification URL List from the configuration
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/notifications -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            HTTP/1.0 200
-            {
-                'notification_urls': ["notification-urls-list"]
-            }
-        @apiName Get
-        @apiGroup Notifications
-        """
 
         notification_urls = self.datastore.data.get('settings', {}).get('application', {}).get('notification_urls', [])        
 
@@ -33,18 +21,10 @@ class Notifications(Resource):
                }, 200
     
     @auth.check_token
+    @validate_openapi_request('addNotifications')
     @expects_json(schema_create_notification_urls)
     def post(self):
-        """
+        """Create Notification URLs."""
-        @api {post} /api/v1/notifications Create Notification URLs
-        @apiDescription Add one or more notification URLs from the configuration
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/notifications/batch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"notification_urls": ["url1", "url2"]}'
-        @apiName CreateBatch
-        @apiGroup Notifications
-        @apiSuccess (201) {Object[]} notification_urls List of added notification URLs
-        @apiError (400) {String} Invalid input
-        """
 
         json_data = request.get_json()
         notification_urls = json_data.get("notification_urls", [])
@@ -69,18 +49,10 @@ class Notifications(Resource):
         return {'notification_urls': added_urls}, 201
     
     @auth.check_token
+    @validate_openapi_request('replaceNotifications')
     @expects_json(schema_create_notification_urls)
     def put(self):
-        """
+        """Replace Notification URLs."""
-        @api {put} /api/v1/notifications Replace Notification URLs
-        @apiDescription Replace all notification URLs with the provided list (can be empty)
-        @apiExample {curl} Example usage:
-            curl -X PUT http://localhost:5000/api/v1/notifications -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"notification_urls": ["url1", "url2"]}'
-        @apiName Replace
-        @apiGroup Notifications
-        @apiSuccess (200) {Object[]} notification_urls List of current notification URLs
-        @apiError (400) {String} Invalid input
-        """
         json_data = request.get_json()
         notification_urls = json_data.get("notification_urls", [])
 
@@ -100,19 +72,10 @@ class Notifications(Resource):
         return {'notification_urls': clean_urls}, 200
         
     @auth.check_token
+    @validate_openapi_request('deleteNotifications')
     @expects_json(schema_delete_notification_urls)
     def delete(self):
-        """
+        """Delete Notification URLs."""
-        @api {delete} /api/v1/notifications Delete Notification URLs
-        @apiDescription Deletes one or more notification URLs from the configuration
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/notifications -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"notification_urls": ["url1", "url2"]}'
-        @apiParam {String[]} notification_urls The notification URLs to delete.
-        @apiName Delete
-        @apiGroup Notifications
-        @apiSuccess (204) {String} OK Deleted
-        @apiError (400) {String} No matching notification URLs found.
-        """
 
         json_data = request.get_json()
         urls_to_delete = json_data.get("notification_urls", [])

changedetectionio/api/Search.py

@@ -1,6 +1,6 @@
 from flask_restful import Resource, abort
 from flask import request
-from . import auth
+from . import auth, validate_openapi_request
 
 class Search(Resource):
     def __init__(self, **kwargs):
@@ -8,21 +8,9 @@ class Search(Resource):
         self.datastore = kwargs['datastore']
 
     @auth.check_token
+    @validate_openapi_request('searchWatches')
     def get(self):
-        """
+        """Search for watches by URL or title text."""
-        @api {get} /api/v1/search Search for watches
-        @apiDescription Search watches by URL or title text
-        @apiExample {curl} Example usage:
-            curl "http://localhost:5000/api/v1/search?q=https://example.com/page1" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            curl "http://localhost:5000/api/v1/search?q=https://example.com/page1?tag=Favourites" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            curl "http://localhost:5000/api/v1/search?q=https://example.com?partial=true" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-        @apiName Search
-        @apiGroup Watch Management
-        @apiQuery {String} q Search query to match against watch URLs and titles
-        @apiQuery {String} [tag] Optional name of tag to limit results (name not UUID)
-        @apiQuery {String} [partial] Allow partial matching of URL query
-        @apiSuccess (200) {Object} JSON Object containing matched watches
-        """
         query = request.args.get('q', '').strip()
         tag_limit = request.args.get('tag', '').strip()
         from changedetectionio.strtobool import strtobool

changedetectionio/api/SystemInfo.py

@@ -1,5 +1,5 @@
 from flask_restful import Resource
-from . import auth
+from . import auth, validate_openapi_request
 
 
 class SystemInfo(Resource):
@@ -9,23 +9,9 @@ class SystemInfo(Resource):
         self.update_q = kwargs['update_q']
 
     @auth.check_token
+    @validate_openapi_request('getSystemInfo')
     def get(self):
-        """
+        """Return system info."""
-        @api {get} /api/v1/systeminfo Return system info
-        @apiDescription Return some info about the current system state
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            HTTP/1.0 200
-            {
-                'queue_size': 10 ,
-                'overdue_watches': ["watch-uuid-list"],
-                'uptime': 38344.55,
-                'watch_count': 800,
-                'version': "0.40.1"
-            }
-        @apiName Get Info
-        @apiGroup System Information
-        """
         import time
         overdue_watches = []
 

changedetectionio/api/Tags.py

@@ -1,39 +1,46 @@
+from changedetectionio import queuedWatchMetaData
+from changedetectionio import worker_handler
 from flask_expects_json import expects_json
 from flask_restful import abort, Resource
+
 from flask import request
 from . import auth
 
 # Import schemas from __init__.py
-from . import schema_tag, schema_create_tag, schema_update_tag
+from . import schema_tag, schema_create_tag, schema_update_tag, validate_openapi_request
 
 
 class Tag(Resource):
     def __init__(self, **kwargs):
         # datastore is a black box dependency
         self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
 
     # Get information about a single tag
     # curl http://localhost:5000/api/v1/tag/<string:uuid>
     @auth.check_token
+    @validate_openapi_request('getTag')
     def get(self, uuid):
-        """
+        """Get data for a single tag/group, toggle notification muting, or recheck all."""
-        @api {get} /api/v1/tag/:uuid Single tag - get data or toggle notification muting.
-        @apiDescription Retrieve tag information and set notification_muted status
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/tag/cc0cfffa-f449-477b-83ea-0caafd1dc091 -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            curl "http://localhost:5000/api/v1/tag/cc0cfffa-f449-477b-83ea-0caafd1dc091?muted=muted" -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-        @apiName Tag
-        @apiGroup Tag
-        @apiParam {uuid} uuid Tag unique ID.
-        @apiQuery {String} [muted] =`muted` or =`unmuted` , Sets the MUTE NOTIFICATIONS state
-        @apiSuccess (200) {String} OK When muted operation OR full JSON object of the tag
-        @apiSuccess (200) {JSON} TagJSON JSON Full JSON object of the tag
-        """
         from copy import deepcopy
         tag = deepcopy(self.datastore.data['settings']['application']['tags'].get(uuid))
         if not tag:
             abort(404, message=f'No tag exists with the UUID of {uuid}')
 
+        if request.args.get('recheck'):
+            # Recheck all, including muted
+            # Get most overdue first
+            i=0
+            for k in sorted(self.datastore.data['watching'].items(), key=lambda item: item[1].get('last_checked', 0)):
+                watch_uuid = k[0]
+                watch = k[1]
+                if not watch['paused'] and tag['uuid'] not in watch['tags']:
+                    continue
+                worker_handler.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))
+                i+=1
+
+            return f"OK, {i} watches queued", 200
+
         if request.args.get('muted', '') == 'muted':
             self.datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = True
             return "OK", 200
@@ -44,16 +51,9 @@ class Tag(Resource):
         return tag
 
     @auth.check_token
+    @validate_openapi_request('deleteTag')
     def delete(self, uuid):
-        """
+        """Delete a tag/group and remove it from all watches."""
-        @api {delete} /api/v1/tag/:uuid Delete a tag and remove it from all watches
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/tag/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-        @apiParam {uuid} uuid Tag unique ID.
-        @apiName DeleteTag
-        @apiGroup Tag
-        @apiSuccess (200) {String} OK Was deleted
-        """
         if not self.datastore.data['settings']['application']['tags'].get(uuid):
             abort(400, message='No tag exists with the UUID of {}'.format(uuid))
 
@@ -68,21 +68,10 @@ class Tag(Resource):
         return 'OK', 204
 
     @auth.check_token
+    @validate_openapi_request('updateTag')
     @expects_json(schema_update_tag)
     def put(self, uuid):
-        """
+        """Update tag information."""
-        @api {put} /api/v1/tag/:uuid Update tag information
-        @apiExample {curl} Example usage:
-            Update (PUT)
-            curl http://localhost:5000/api/v1/tag/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"title": "New Tag Title"}'
-
-        @apiDescription Updates an existing tag using JSON
-        @apiParam {uuid} uuid Tag unique ID.
-        @apiName UpdateTag
-        @apiGroup Tag
-        @apiSuccess (200) {String} OK Was updated
-        @apiSuccess (500) {String} ERR Some other error
-        """
         tag = self.datastore.data['settings']['application']['tags'].get(uuid)
         if not tag:
             abort(404, message='No tag exists with the UUID of {}'.format(uuid))
@@ -94,17 +83,10 @@ class Tag(Resource):
 
 
     @auth.check_token
+    @validate_openapi_request('createTag')
     # Only cares for {'title': 'xxxx'}
     def post(self):
-        """
+        """Create a single tag/group."""
-        @api {post} /api/v1/watch Create a single tag
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"name": "Work related"}'
-        @apiName Create
-        @apiGroup Tag
-        @apiSuccess (200) {String} OK Was created
-        @apiSuccess (500) {String} ERR Some other error
-        """
 
         json_data = request.get_json()
         title = json_data.get("title",'').strip()
@@ -122,28 +104,9 @@ class Tags(Resource):
         self.datastore = kwargs['datastore']
 
     @auth.check_token
+    @validate_openapi_request('listTags')
     def get(self):
-        """
+        """List tags/groups."""
-        @api {get} /api/v1/tags List tags
-        @apiDescription Return list of available tags
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/tags -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            {
-                "cc0cfffa-f449-477b-83ea-0caafd1dc091": {
-                    "title": "Tech News",
-                    "notification_muted": false,
-                    "date_created": 1677103794
-                },
-                "e6f5fd5c-dbfe-468b-b8f3-f9d6ff5ad69b": {
-                    "title": "Shopping",
-                    "notification_muted": true,
-                    "date_created": 1676662819
-                }
-            }
-        @apiName ListTags
-        @apiGroup Tag Management
-        @apiSuccess (200) {String} OK JSON dict
-        """
         result = {}
         for uuid, tag in self.datastore.data['settings']['application']['tags'].items():
             result[uuid] = {

changedetectionio/api/Watch.py

@@ -11,7 +11,40 @@ from . import auth
 import copy
 
 # Import schemas from __init__.py
-from . import schema, schema_create_watch, schema_update_watch
+from . import schema, schema_create_watch, schema_update_watch, validate_openapi_request
+
+
+def validate_time_between_check_required(json_data):
+    """
+    Validate that at least one time interval is specified when not using default settings.
+    Returns None if valid, or error message string if invalid.
+    Defaults to using global settings if time_between_check_use_default is not provided.
+    """
+    # Default to using global settings if not specified
+    use_default = json_data.get('time_between_check_use_default', True)
+
+    # If using default settings, no validation needed
+    if use_default:
+        return None
+
+    # If not using defaults, check if time_between_check exists and has at least one non-zero value
+    time_check = json_data.get('time_between_check')
+    if not time_check:
+        # No time_between_check provided and not using defaults - this is an error
+        return "At least one time interval (weeks, days, hours, minutes, or seconds) must be specified when not using global settings."
+
+    # time_between_check exists, check if it has at least one non-zero value
+    if any([
+        (time_check.get('weeks') or 0) > 0,
+        (time_check.get('days') or 0) > 0,
+        (time_check.get('hours') or 0) > 0,
+        (time_check.get('minutes') or 0) > 0,
+        (time_check.get('seconds') or 0) > 0
+    ]):
+        return None
+
+    # time_between_check exists but all values are 0 or empty - this is an error
+    return "At least one time interval (weeks, days, hours, minutes, or seconds) must be specified when not using global settings."
 
 
 class Watch(Resource):
@@ -25,23 +58,9 @@ class Watch(Resource):
     # @todo - version2 - ?muted and ?paused should be able to be called together, return the watch struct not "OK"
     # ?recheck=true
     @auth.check_token
+    @validate_openapi_request('getWatch')
     def get(self, uuid):
-        """
+        """Get information about a single watch, recheck, pause, or mute."""
-        @api {get} /api/v1/watch/:uuid Single watch - get data, recheck, pause, mute.
-        @apiDescription Retrieve watch information and set muted/paused status
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            curl "http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?muted=unmuted"  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            curl "http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?paused=unpaused"  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-        @apiName Watch
-        @apiGroup Watch
-        @apiParam {uuid} uuid Watch unique ID.
-        @apiQuery {Boolean} [recheck] Recheck this watch `recheck=1`
-        @apiQuery {String} [paused] =`paused` or =`unpaused` , Sets the PAUSED state
-        @apiQuery {String} [muted] =`muted` or =`unmuted` , Sets the MUTE NOTIFICATIONS state
-        @apiSuccess (200) {String} OK When paused/muted/recheck operation OR full JSON object of the watch
-        @apiSuccess (200) {JSON} WatchJSON JSON Full JSON object of the watch
-        """
         from copy import deepcopy
         watch = deepcopy(self.datastore.data['watching'].get(uuid))
         if not watch:
@@ -69,19 +88,14 @@ class Watch(Resource):
         # attr .last_changed will check for the last written text snapshot on change
         watch['last_changed'] = watch.last_changed
         watch['viewed'] = watch.viewed
+        watch['link'] = watch.link,
+
         return watch
 
     @auth.check_token
+    @validate_openapi_request('deleteWatch')
     def delete(self, uuid):
-        """
+        """Delete a watch and related history."""
-        @api {delete} /api/v1/watch/:uuid Delete a watch and related history
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-        @apiParam {uuid} uuid Watch unique ID.
-        @apiName Delete
-        @apiGroup Watch
-        @apiSuccess (200) {String} OK Was deleted
-        """
         if not self.datastore.data['watching'].get(uuid):
             abort(400, message='No watch exists with the UUID of {}'.format(uuid))
 
@@ -89,21 +103,10 @@ class Watch(Resource):
         return 'OK', 204
 
     @auth.check_token
+    @validate_openapi_request('updateWatch')
     @expects_json(schema_update_watch)
     def put(self, uuid):
-        """
+        """Update watch information."""
-        @api {put} /api/v1/watch/:uuid Update watch information
-        @apiExample {curl} Example usage:
-            Update (PUT)
-            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}'
-
-        @apiDescription Updates an existing watch using JSON, accepts the same structure as returned in <a href="#api-Watch-Watch">get single watch information</a>
-        @apiParam {uuid} uuid Watch unique ID.
-        @apiName Update a watch
-        @apiGroup Watch
-        @apiSuccess (200) {String} OK Was updated
-        @apiSuccess (500) {String} ERR Some other error
-        """
         watch = self.datastore.data['watching'].get(uuid)
         if not watch:
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
@@ -113,6 +116,11 @@ class Watch(Resource):
             if not request.json.get('proxy') in plist:
                 return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
 
+        # Validate time_between_check when not using defaults
+        validation_error = validate_time_between_check_required(request.json)
+        if validation_error:
+            return validation_error, 400
+
         watch.update(request.json)
 
         return "OK", 200
@@ -126,22 +134,9 @@ class WatchHistory(Resource):
     # Get a list of available history for a watch by UUID
     # curl http://localhost:5000/api/v1/watch/<string:uuid>/history
     @auth.check_token
+    @validate_openapi_request('getWatchHistory')
     def get(self, uuid):
-        """
+        """Get a list of all historical snapshots available for a watch."""
-        @api {get} /api/v1/watch/<string:uuid>/history Get a list of all historical snapshots available for a watch
-        @apiDescription Requires `uuid`, returns list
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
-            {
-                "1676649279": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/cb7e9be8258368262246910e6a2a4c30.txt",
-                "1677092785": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/e20db368d6fc633e34f559ff67bb4044.txt",
-                "1677103794": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/02efdd37dacdae96554a8cc85dc9c945.txt"
-            }
-        @apiName Get list of available stored snapshots for watch
-        @apiGroup Watch History
-        @apiSuccess (200) {String} OK
-        @apiSuccess (404) {String} ERR Not found
-        """
         watch = self.datastore.data['watching'].get(uuid)
         if not watch:
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
@@ -154,18 +149,9 @@ class WatchSingleHistory(Resource):
         self.datastore = kwargs['datastore']
 
     @auth.check_token
+    @validate_openapi_request('getWatchSnapshot')
     def get(self, uuid, timestamp):
-        """
+        """Get single snapshot from watch."""
-        @api {get} /api/v1/watch/<string:uuid>/history/<int:timestamp> Get single snapshot from watch
-        @apiDescription Requires watch `uuid` and `timestamp`. `timestamp` of "`latest`" for latest available snapshot, or <a href="#api-Watch_History-Get_list_of_available_stored_snapshots_for_watch">use the list returned here</a>
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
-        @apiName Get single snapshot content
-        @apiGroup Watch History
-        @apiParam {String} [html]       Optional Set to =1 to return the last HTML (only stores last 2 snapshots, use `latest` as timestamp)
-        @apiSuccess (200) {String} OK
-        @apiSuccess (404) {String} ERR Not found
-        """
         watch = self.datastore.data['watching'].get(uuid)
         if not watch:
             abort(404, message=f"No watch exists with the UUID of {uuid}")
@@ -197,17 +183,9 @@ class WatchFavicon(Resource):
         self.datastore = kwargs['datastore']
 
     @auth.check_token
+    @validate_openapi_request('getWatchFavicon')
     def get(self, uuid):
-        """
+        """Get favicon for a watch."""
-        @api {get} /api/v1/watch/<string:uuid>/favicon Get Favicon for a watch
-        @apiDescription Requires watch `uuid`
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/favicon -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-        @apiName Get latest Favicon
-        @apiGroup Watch History
-        @apiSuccess (200) {String} OK
-        @apiSuccess (404) {String} ERR Not found
-        """
         watch = self.datastore.data['watching'].get(uuid)
         if not watch:
             abort(404, message=f"No watch exists with the UUID of {uuid}")
@@ -240,18 +218,10 @@ class CreateWatch(Resource):
         self.update_q = kwargs['update_q']
 
     @auth.check_token
+    @validate_openapi_request('createWatch')
     @expects_json(schema_create_watch)
     def post(self):
-        """
+        """Create a single watch."""
-        @api {post} /api/v1/watch Create a single watch
-        @apiDescription Requires atleast `url` set, can accept the same structure as <a href="#api-Watch-Watch">get single watch information</a> to create.
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
-        @apiName Create
-        @apiGroup Watch
-        @apiSuccess (200) {String} OK Was created
-        @apiSuccess (500) {String} ERR Some other error
-        """
 
         json_data = request.get_json()
         url = json_data['url'].strip()
@@ -266,6 +236,11 @@ class CreateWatch(Resource):
             if not json_data.get('proxy') in plist:
                 return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
 
+        # Validate time_between_check when not using defaults
+        validation_error = validate_time_between_check_required(json_data)
+        if validation_error:
+            return validation_error, 400
+
         extras = copy.deepcopy(json_data)
 
         # Because we renamed 'tag' to 'tags' but don't want to change the API (can do this in v2 of the API)
@@ -284,35 +259,9 @@ class CreateWatch(Resource):
             return "Invalid or unsupported URL", 400
 
     @auth.check_token
+    @validate_openapi_request('listWatches')
     def get(self):
-        """
+        """List watches."""
-        @api {get} /api/v1/watch List watches
-        @apiDescription Return concise list of available watches and some very basic info
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            {
-                "6a4b7d5c-fee4-4616-9f43-4ac97046b595": {
-                    "last_changed": 1677103794,
-                    "last_checked": 1677103794,
-                    "last_error": false,
-                    "title": "",
-                    "url": "http://www.quotationspage.com/random.php"
-                },
-                "e6f5fd5c-dbfe-468b-b8f3-f9d6ff5ad69b": {
-                    "last_changed": 0,
-                    "last_checked": 1676662819,
-                    "last_error": false,
-                    "title": "QuickLook",
-                    "url": "https://github.com/QL-Win/QuickLook/tags"
-                }
-            }
-
-        @apiParam {String} [recheck_all]       Optional Set to =1 to force recheck of all watches
-        @apiParam {String} [tag]               Optional name of tag to limit results
-        @apiName ListWatches
-        @apiGroup Watch Management
-        @apiSuccess (200) {String} OK JSON dict
-        """
         list = {}
 
         tag_limit = request.args.get('tag', '').lower()
@@ -326,6 +275,8 @@ class CreateWatch(Resource):
                 'last_changed': watch.last_changed,
                 'last_checked': watch['last_checked'],
                 'last_error': watch['last_error'],
+                'link': watch.link,
+                'page_title': watch['page_title'],
                 'title': watch['title'],
                 'url': watch['url'],
                 'viewed': watch.viewed

changedetectionio/api/__init__.py

@@ -1,4 +1,10 @@
 import copy
+import yaml
+import functools
+from flask import request, abort
+from loguru import logger
+from openapi_core import OpenAPI
+from openapi_core.contrib.flask import FlaskOpenAPIRequest
 from . import api_schema
 from ..model import watch_base
 
@@ -8,6 +14,7 @@ schema = api_schema.build_watch_json_schema(watch_base_config)
 
 schema_create_watch = copy.deepcopy(schema)
 schema_create_watch['required'] = ['url']
+del schema_create_watch['properties']['last_viewed']
 
 schema_update_watch = copy.deepcopy(schema)
 schema_update_watch['additionalProperties'] = False
@@ -25,9 +32,47 @@ schema_create_notification_urls['required'] = ['notification_urls']
 schema_delete_notification_urls = copy.deepcopy(schema_notification_urls)
 schema_delete_notification_urls['required'] = ['notification_urls']
 
+@functools.cache
+def get_openapi_spec():
+    import os
+    spec_path = os.path.join(os.path.dirname(__file__), '../../docs/api-spec.yaml')
+    with open(spec_path, 'r') as f:
+        spec_dict = yaml.safe_load(f)
+    _openapi_spec = OpenAPI.from_dict(spec_dict)
+    return _openapi_spec
+
+def validate_openapi_request(operation_id):
+    """Decorator to validate incoming requests against OpenAPI spec."""
+    def decorator(f):
+        @functools.wraps(f)
+        def wrapper(*args, **kwargs):
+            try:
+                # Skip OpenAPI validation for GET requests since they don't have request bodies
+                if request.method.upper() != 'GET':
+                    spec = get_openapi_spec()
+                    openapi_request = FlaskOpenAPIRequest(request)
+                    result = spec.unmarshal_request(openapi_request)
+                    if result.errors:
+                        from werkzeug.exceptions import BadRequest
+                        error_details = []
+                        for error in result.errors:
+                            error_details.append(str(error))
+                        raise BadRequest(f"OpenAPI validation failed: {error_details}")
+            except BadRequest:
+                # Re-raise BadRequest exceptions (validation failures)
+                raise
+            except Exception as e:
+                # If OpenAPI spec loading fails, log but don't break existing functionality
+                logger.critical(f"OpenAPI validation warning for {operation_id}: {e}")
+                abort(500)
+            return f(*args, **kwargs)
+        return wrapper
+    return decorator
+
 # Import all API resources
 from .Watch import Watch, WatchHistory, WatchSingleHistory, CreateWatch, WatchFavicon
 from .Tags import Tags, Tag
 from .Import import Import
 from .SystemInfo import SystemInfo
 from .Notifications import Notifications
+

changedetectionio/api/api_schema.py

@@ -78,6 +78,13 @@ def build_watch_json_schema(d):
               ]:
         schema['properties'][v]['anyOf'].append({'type': 'string', "maxLength": 5000})
 
+    for v in ['last_viewed']:
+        schema['properties'][v] = {
+            "type": "integer",
+            "description": "Unix timestamp in seconds of the last time the watch was viewed.",
+            "minimum": 0
+        }
+
     # None or Boolean
     schema['properties']['track_ldjson_price_data']['anyOf'].append({'type': 'boolean'})
 
@@ -112,6 +119,12 @@ def build_watch_json_schema(d):
 
     schema['properties']['time_between_check'] = build_time_between_check_json_schema()
 
+    schema['properties']['time_between_check_use_default'] = {
+        "type": "boolean",
+        "default": True,
+        "description": "Whether to use global settings for time between checks - defaults to true if not set"
+    }
+
     schema['properties']['browser_steps'] = {
         "anyOf": [
             {

changedetectionio/async_update_worker.py

@@ -7,6 +7,7 @@ from changedetectionio.flask_app import watch_check_update
 import asyncio
 import importlib
 import os
+import queue
 import time
 
 from loguru import logger
@@ -37,13 +38,23 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
         watch = None
 
         try:
-            # Use asyncio wait_for to make queue.get() cancellable
+            # Use native janus async interface - no threads needed!
-            queued_item_data = await asyncio.wait_for(q.get(), timeout=1.0)
+            queued_item_data = await asyncio.wait_for(q.async_get(), timeout=1.0)
+            
         except asyncio.TimeoutError:
             # No jobs available, continue loop
             continue
         except Exception as e:
-            logger.error(f"Worker {worker_id} error getting queue item: {e}")
+            logger.critical(f"CRITICAL: Worker {worker_id} failed to get queue item: {type(e).__name__}: {e}")
+            
+            # Log queue health for debugging
+            try:
+                queue_size = q.qsize()
+                is_empty = q.empty()
+                logger.critical(f"CRITICAL: Worker {worker_id} queue health - size: {queue_size}, empty: {is_empty}")
+            except Exception as health_e:
+                logger.critical(f"CRITICAL: Worker {worker_id} queue health check failed: {health_e}")
+            
             await asyncio.sleep(0.1)
             continue
         
@@ -299,15 +310,6 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
                     continue
 
                 if process_changedetection_results:
-                    # Extract title if needed
-                    if datastore.data['settings']['application'].get('extract_title_as_title') or watch['extract_title_as_title']:
-                        if not watch['title'] or not len(watch['title']):
-                            try:
-                                update_obj['title'] = html_tools.extract_element(find='title', html_content=update_handler.fetcher.content)
-                                logger.info(f"UUID: {uuid} Extract <title> updated title to '{update_obj['title']}")
-                            except Exception as e:
-                                logger.warning(f"UUID: {uuid} Extract <title> as watch title was enabled, but couldn't find a <title>.")
-
                     try:
                         datastore.update_watch(uuid=uuid, update_obj=update_obj)
 
@@ -346,6 +348,14 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
                 # Always record attempt count
                 count = watch.get('check_count', 0) + 1
 
+                # Always record page title (used in notifications, and can change even when the content is the same)
+                try:
+                    page_title = html_tools.extract_title(data=update_handler.fetcher.content)
+                    logger.debug(f"UUID: {uuid} Page <title> is '{page_title}'")
+                    datastore.update_watch(uuid=uuid, update_obj={'page_title': page_title})
+                except Exception as e:
+                    logger.warning(f"UUID: {uuid} Exception when extracting <title> - {str(e)}")
+
                 # Record server header
                 try:
                     server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]

changedetectionio/blueprint/rss/blueprint.py

@@ -108,10 +108,13 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
                 fe.link(link=diff_link)
 
-                # @todo watch should be a getter - watch.get('title') (internally if URL else..)
+                # Same logic as watch-overview.html
+                if datastore.data['settings']['application']['ui'].get('use_page_title_in_list') or watch.get('use_page_title_in_list'):
+                    watch_label = watch.label
+                else:
+                    watch_label = watch.get('url')
 
-                watch_title = watch.get('title') if watch.get('title') else watch.get('url')
+                fe.title(title=watch_label)
-                fe.title(title=watch_title)
                 try:
 
                     html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]),
@@ -127,7 +130,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 # @todo User could decide if <link> goes to the diff page, or to the watch link
                 rss_template = "<html><body>\n<h4><a href=\"{{watch_url}}\">{{watch_title}}</a></h4>\n<p>{{html_diff}}</p>\n</body></html>\n"
 
-                content = jinja_render(template_str=rss_template, watch_title=watch_title, html_diff=html_diff, watch_url=watch.link)
+                content = jinja_render(template_str=rss_template, watch_title=watch_label, html_diff=html_diff, watch_url=watch.link)
 
                 # Out of range chars could also break feedgen
                 if scan_invalid_chars_in_rss(content):

changedetectionio/blueprint/settings/templates/settings.html

@@ -1,7 +1,7 @@
 {% extends 'base.html' %}
 
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, render_ternary_field %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script>
     const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="global-settings")}}";
@@ -75,18 +75,10 @@
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }}
                     </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.application.form.pager_size) }}
-                        <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
-                    </div>
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.rss_content_format) }}
                         <span class="pure-form-message-inline">Love RSS? Does your reader support HTML? Set it here</span>
                     </div>
-                    <div class="pure-control-group">
-                        {{ render_checkbox_field(form.application.form.extract_title_as_title) }}
-                        <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
-                    </div>
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}
                         <span class="pure-form-message-inline">When a request returns no content, or the HTML does not contain any text, is this considered a change?</span>
@@ -203,7 +195,7 @@ nav
 
             <div class="tab-pane-inner" id="api">
                 <h4>API Access</h4>
-                <p>Drive your changedetection.io via API, More about <a href="https://github.com/dgtlmoon/changedetection.io/wiki/API-Reference">API access here</a></p>
+                <p>Drive your changedetection.io via API, More about <a href="https://changedetection.io/docs/api_v1/index.html">API access and examples here</a>.</p>
 
                 <div class="pure-control-group">
                     {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
@@ -260,6 +252,13 @@ nav
                     {{ render_checkbox_field(form.application.form.ui.form.favicons_enabled, class="") }}
                     <span class="pure-form-message-inline">Enable or Disable Favicons next to the watch list</span>
                 </div>
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.ui.use_page_title_in_list) }}
+                </div>
+                <div class="pure-control-group">
+                    {{ render_field(form.application.form.pager_size) }}
+                    <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
+                </div>
 
             </div>
             <div class="tab-pane-inner" id="proxies">
@@ -324,8 +323,8 @@ nav
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
-                    <a href="{{url_for('watchlist.index')}}" class="pure-button button-small button-cancel">Back</a>
+                    <a href="{{url_for('watchlist.index')}}" class="pure-button button-cancel">Back</a>
-                    <a href="{{url_for('ui.clear_all_history')}}" class="pure-button button-small button-error">Clear Snapshot History</a>
+                    <a href="{{url_for('ui.clear_all_history')}}" class="pure-button button-error">Clear Snapshot History</a>
                 </div>
             </div>
         </form>

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_ternary_field %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script>
     const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="group-settings")}}";
@@ -64,7 +64,7 @@
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div  class="pure-control-group inline-radio">
-                      {{ render_checkbox_field(form.notification_muted) }}
+                      {{ render_ternary_field(form.notification_muted, BooleanField=True) }}
                     </div>
                     {% if 1 %}
                     <div class="pure-control-group inline-radio">

changedetectionio/blueprint/ui/edit.py

@@ -242,6 +242,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                 'available_timezones': sorted(available_timezones()),
                 'browser_steps_config': browser_step_ui_config,
                 'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
+                'extra_classes': 'checking-now' if worker_handler.is_watch_running(uuid) else '',
                 'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
                 'extra_processor_config': form.extra_tab_content(),
                 'extra_title': f" - Edit - {watch.label}",

changedetectionio/blueprint/ui/templates/edit.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, playwright_warning, only_playwright_type_watches_warning, render_conditions_fieldlist_of_formfields_as_table %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, playwright_warning, only_playwright_type_watches_warning, render_conditions_fieldlist_of_formfields_as_table, render_ternary_field %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
@@ -72,15 +72,16 @@
                         <div class="pure-form-message">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></div>
                         <div class="pure-form-message">Variables are supported in the URL (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a>).</div>
                     </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.tags) }}
+                        <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
+                    </div>
                     <div class="pure-control-group inline-radio">
                         {{ render_field(form.processor) }}
                     </div>
                     <div class="pure-control-group">
-                        {{ render_field(form.title, class="m-d") }}
+                        {{ render_field(form.title, class="m-d", placeholder=watch.label) }}
-                    </div>
+                        <span class="pure-form-message-inline">Automatically uses the page title if found, you can also use your own title/description here</span>
-                    <div class="pure-control-group">
-                        {{ render_field(form.tags) }}
-                        <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
                     </div>
                     <div class="pure-control-group time-between-check border-fieldset">
 
@@ -101,15 +102,16 @@
                         </div>
 <br>
               </div>
-                    <div class="pure-control-group">
+
-                        {{ render_checkbox_field(form.extract_title_as_title) }}
-                    </div>
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.filter_failure_notification_send) }}
                         <span class="pure-form-message-inline">
                          Sends a notification when the filter can no longer be seen on the page, good for knowing when the page changed and your filter will not work anymore.
                         </span>
                     </div>
+                    <div class="pure-control-group">
+                        {{ render_ternary_field(form.use_page_title_in_list) }}
+                    </div>
                 </fieldset>
             </div>
 
@@ -262,7 +264,7 @@ Math: {{ 1 + 1 }}") }}
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div  class="pure-control-group inline-radio">
-                      {{ render_checkbox_field(form.notification_muted) }}
+                      {{ render_ternary_field(form.notification_muted, BooleanField=true) }}
                     </div>
                     {% if watch_needs_selenium_or_playwright %}
                     <div class="pure-control-group inline-radio">
@@ -469,11 +471,11 @@ Math: {{ 1 + 1 }}") }}
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
                     <a href="{{url_for('ui.form_delete', uuid=uuid)}}"
-                       class="pure-button button-small button-error ">Delete</a>
+                       class="pure-button button-error ">Delete</a>
                     {% if watch.history_n %}<a href="{{url_for('ui.clear_watch_history', uuid=uuid)}}"
-                       class="pure-button button-small button-error ">Clear History</a>{% endif %}
+                       class="pure-button button-error">Clear History</a>{% endif %}
                     <a href="{{url_for('ui.form_clone', uuid=uuid)}}"
-                       class="pure-button button-small ">Clone &amp; Edit</a>
+                       class="pure-button">Clone &amp; Edit</a>
                 </div>
             </div>
         </form>

changedetectionio/blueprint/watchlist/__init__.py

@@ -44,12 +44,16 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
         with_errors = request.args.get('with_errors') == "1"
+        unread_only = request.args.get('unread') == "1"
         errored_count = 0
         search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
         for uuid, watch in datastore.data['watching'].items():
             if with_errors and not watch.get('last_error'):
                 continue
 
+            if unread_only and (watch.viewed or watch.last_changed == 0) :
+                continue
+
             if active_tag_uuid and not active_tag_uuid in watch['tags']:
                     continue
             if watch.get('last_error'):

changedetectionio/blueprint/watchlist/templates/watch-overview.html

@@ -118,7 +118,8 @@ document.addEventListener('DOMContentLoaded', function() {
                 {%- set checking_now = is_checking_now(watch) -%}
                 {%- set history_n = watch.history_n -%}
                 {%- set favicon = watch.get_favicon_filename() -%}
-                {#  Mirror in changedetectionio/static/js/realtime.js for the frontend #}
+                {%- set system_use_url_watchlist = datastore.data['settings']['application']['ui'].get('use_page_title_in_list')  -%}
+                {#  Class settings mirrored in changedetectionio/static/js/realtime.js for the frontend #}
                 {%- set row_classes = [
                     loop.cycle('pure-table-odd', 'pure-table-even'),
                     'processor-' ~ watch['processor'],
@@ -133,7 +134,8 @@ document.addEventListener('DOMContentLoaded', function() {
                     'checking-now' if checking_now else '',
                     'notification_muted' if watch.notification_muted else '',
                     'single-history' if history_n == 1 else '',
-                    'multiple-history' if history_n >= 2 else '',                    
+                    'multiple-history' if history_n >= 2 else '',
+                    'use-html-title' if system_use_url_watchlist else 'no-html-title',
                 ] -%}
             <tr id="{{ watch.uuid }}" data-watch-uuid="{{ watch.uuid }}" class="{{ row_classes | reject('equalto', '') | join(' ') }}">
                 <td class="inline checkbox-uuid" ><div><input name="uuids"  type="checkbox" value="{{ watch.uuid}} " > <span class="counter-i">{{ loop.index+pagination.skip }}</span></div></td>
@@ -155,7 +157,12 @@ document.addEventListener('DOMContentLoaded', function() {
                     {%  endif %}
                         <div>
                         <span class="watch-title">
-                            {{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}&nbsp;<a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}">&nbsp;</a>
+                            {% if system_use_url_watchlist or watch.get('use_page_title_in_list') %}
+                                {{watch.label}}
+                            {% else %}
+                                {{watch.url}}
+                            {% endif %}
+                           <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}">&nbsp;</a>
                         </span>
                             <div class="error-text" style="display:none;">{{ watch.compile_error_texts(has_proxies=datastore.proxy_list) }}</div>
                             {%- if watch['processor'] == 'text_json_diff'  -%}
@@ -245,6 +252,9 @@ document.addEventListener('DOMContentLoaded', function() {
                 <a href="{{url_for('ui.mark_all_viewed', tag=active_tag_uuid) }}" class="pure-button button-tag " id="mark-all-viewed">Mark all viewed in '{{active_tag.title}}'</a>
             </li>
         {%-  endif -%}
+            <li id="post-list-unread" class="{%- if has_unviewed -%}has-unviewed{%- endif -%}" style="display: none;" >
+                <a href="{{url_for('watchlist.index', unread=1, tag=request.args.get('tag')) }}" class="pure-button button-tag">Unread</a>
+            </li>
             <li>
                <a href="{{ url_for('ui.form_watch_checknow', tag=active_tag_uuid, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag" id="recheck-all">Recheck
                 all {% if active_tag_uuid %}  in '{{active_tag.title}}'{%endif%}</a>

changedetectionio/content_fetchers/base.py

@@ -70,15 +70,17 @@ class Fetcher():
 
     @abstractmethod
     async def run(self,
-            url,
+                  fetch_favicon=True,
-            timeout,
+                  current_include_filters=None,
-            request_headers,
+                  empty_pages_are_a_change=False,
-            request_body,
+                  ignore_status_codes=False,
-            request_method,
+                  is_binary=False,
-            ignore_status_codes=False,
+                  request_body=None,
-            current_include_filters=None,
+                  request_headers=None,
-            is_binary=False,
+                  request_method=None,
-            empty_pages_are_a_change=False):
+                  timeout=None,
+                  url=None,
+                  ):
         # Should set self.error, self.status_code and self.content
         pass
 

changedetectionio/content_fetchers/playwright.py

@@ -143,15 +143,17 @@ class fetcher(Fetcher):
             f.write(content)
 
     async def run(self,
-            url,
+                  fetch_favicon=True,
-            timeout,
+                  current_include_filters=None,
-            request_headers,
+                  empty_pages_are_a_change=False,
-            request_body,
+                  ignore_status_codes=False,
-            request_method,
+                  is_binary=False,
-            ignore_status_codes=False,
+                  request_body=None,
-            current_include_filters=None,
+                  request_headers=None,
-            is_binary=False,
+                  request_method=None,
-            empty_pages_are_a_change=False):
+                  timeout=None,
+                  url=None,
+                  ):
 
         from playwright.async_api import async_playwright
         import playwright._impl._errors
@@ -234,11 +236,12 @@ class fetcher(Fetcher):
                 await browser.close()
                 raise PageUnloadable(url=url, status_code=None, message=str(e))
 
-            try:
+            if fetch_favicon:
-                self.favicon_blob = await self.page.evaluate(FAVICON_FETCHER_JS)
+                try:
-                await self.page.request_gc()
+                    self.favicon_blob = await self.page.evaluate(FAVICON_FETCHER_JS)
-            except Exception as e:
+                    await self.page.request_gc()
-                logger.error(f"Error fetching FavIcon info {str(e)}, continuing.")
+                except Exception as e:
+                    logger.error(f"Error fetching FavIcon info {str(e)}, continuing.")
 
             if self.status_code != 200 and not ignore_status_codes:
                 screenshot = await capture_full_page_async(self.page)

changedetectionio/content_fetchers/puppeteer.py

@@ -145,15 +145,16 @@ class fetcher(Fetcher):
     #         f.write(content)
 
     async def fetch_page(self,
-                         url,
-                         timeout,
-                         request_headers,
-                         request_body,
-                         request_method,
-                         ignore_status_codes,
                          current_include_filters,
+                         empty_pages_are_a_change,
+                         fetch_favicon,
+                         ignore_status_codes,
                          is_binary,
-                         empty_pages_are_a_change
+                         request_body,
+                         request_headers,
+                         request_method,
+                         timeout,
+                         url,
                          ):
         import re
         self.delete_browser_steps_screenshots()
@@ -181,6 +182,9 @@ class fetcher(Fetcher):
 
         # more reliable is to just request a new page
         self.page = await browser.newPage()
+        
+        # Add console handler to capture console.log from favicon fetcher
+        #self.page.on('console', lambda msg: logger.debug(f"Browser console [{msg.type}]: {msg.text}"))
 
         if '--window-size' in self.browser_connection_url:
             # Be sure the viewport is always the window-size, this is often not the same thing
@@ -290,10 +294,11 @@ class fetcher(Fetcher):
             await browser.close()
             raise PageUnloadable(url=url, status_code=None, message=str(e))
 
-        try:
+        if fetch_favicon:
-            self.favicon_blob = await self.page.evaluate(FAVICON_FETCHER_JS)
+            try:
-        except Exception as e:
+                self.favicon_blob = await self.page.evaluate(FAVICON_FETCHER_JS)
-            logger.error(f"Error fetching FavIcon info {str(e)}, continuing.")
+            except Exception as e:
+                logger.error(f"Error fetching FavIcon info {str(e)}, continuing.")
 
         if self.status_code != 200 and not ignore_status_codes:
             screenshot = await capture_full_page(page=self.page)
@@ -346,8 +351,18 @@ class fetcher(Fetcher):
     async def main(self, **kwargs):
         await self.fetch_page(**kwargs)
 
-    async def run(self, url, timeout, request_headers, request_body, request_method, ignore_status_codes=False,
+    async def run(self,
-            current_include_filters=None, is_binary=False, empty_pages_are_a_change=False):
+                  fetch_favicon=True,
+                  current_include_filters=None,
+                  empty_pages_are_a_change=False,
+                  ignore_status_codes=False,
+                  is_binary=False,
+                  request_body=None,
+                  request_headers=None,
+                  request_method=None,
+                  timeout=None,
+                  url=None,
+                  ):
 
         #@todo make update_worker async which could run any of these content_fetchers within memory and time constraints
         max_time = int(os.getenv('PUPPETEER_MAX_PROCESSING_TIMEOUT_SECONDS', 180))
@@ -355,16 +370,17 @@ class fetcher(Fetcher):
         # Now we run this properly in async context since we're called from async worker
         try:
             await asyncio.wait_for(self.main(
-                url=url,
-                timeout=timeout,
-                request_headers=request_headers,
-                request_body=request_body,
-                request_method=request_method,
-                ignore_status_codes=ignore_status_codes,
                 current_include_filters=current_include_filters,
+                empty_pages_are_a_change=empty_pages_are_a_change,
+                fetch_favicon=fetch_favicon,
+                ignore_status_codes=ignore_status_codes,
                 is_binary=is_binary,
-                empty_pages_are_a_change=empty_pages_are_a_change
+                request_body=request_body,
-            ), timeout=max_time)
+                request_headers=request_headers,
+                request_method=request_method,
+                timeout=timeout,
+                url=url,
+            ), timeout=max_time
+            )
         except asyncio.TimeoutError:
-            raise(BrowserFetchTimedOut(msg=f"Browser connected but was unable to process the page in {max_time} seconds."))
+            raise (BrowserFetchTimedOut(msg=f"Browser connected but was unable to process the page in {max_time} seconds."))
-

changedetectionio/content_fetchers/requests.py

@@ -104,15 +104,17 @@ class fetcher(Fetcher):
         self.raw_content = r.content
 
     async def run(self,
-            url,
+                  fetch_favicon=True,
-            timeout,
+                  current_include_filters=None,
-            request_headers,
+                  empty_pages_are_a_change=False,
-            request_body,
+                  ignore_status_codes=False,
-            request_method,
+                  is_binary=False,
-            ignore_status_codes=False,
+                  request_body=None,
-            current_include_filters=None,
+                  request_headers=None,
-            is_binary=False,
+                  request_method=None,
-            empty_pages_are_a_change=False):
+                  timeout=None,
+                  url=None,
+                  ):
         """Async wrapper that runs the synchronous requests code in a thread pool"""
         
         loop = asyncio.get_event_loop()

changedetectionio/content_fetchers/res/favicon-fetcher.js

@@ -1,79 +1,101 @@
 (async () => {
-  const links = Array.from(document.querySelectorAll(
+  // Define the function inside the IIFE for console testing
-    'link[rel~="apple-touch-icon"], link[rel~="icon"]'
+  window.getFaviconAsBlob = async function() {
-  ));
+    const links = Array.from(document.querySelectorAll(
+      'link[rel~="apple-touch-icon"], link[rel~="icon"]'
+    ));
 
-  const icons = links.map(link => {
+    const icons = links.map(link => {
-    const sizesStr = link.getAttribute('sizes');
+      const sizesStr = link.getAttribute('sizes');
-    let size = 0;
+      let size = 0;
-    if (sizesStr) {
+      if (sizesStr) {
-      const [w] = sizesStr.split('x').map(Number);
+        const [w] = sizesStr.split('x').map(Number);
-      if (!isNaN(w)) size = w;
+        if (!isNaN(w)) size = w;
-    } else {
+      } else {
-      size = 16;
+        size = 16;
-    }
+      }
-    return {
+      return {
-      size,
+        size,
-      rel: link.getAttribute('rel'),
+        rel: link.getAttribute('rel'),
-      href: link.href
+        href: link.href,
-    };
+        hasSizes: !!sizesStr
-  });
+      };
-
-  // If no icons found, add fallback favicon.ico
-  if (icons.length === 0) {
-    icons.push({
-      size: 16,
-      rel: 'icon',
-      href: '/favicon.ico'
     });
-  }
 
-  // sort preference
+    // If no icons found, add fallback favicon.ico
-  icons.sort((a, b) => {
+    if (icons.length === 0) {
-    const isAppleA = /apple-touch-icon/.test(a.rel);
+      icons.push({
-    const isAppleB = /apple-touch-icon/.test(b.rel);
+        size: 16,
-    if (isAppleA && !isAppleB) return -1;
+        rel: 'icon',
-    if (!isAppleA && isAppleB) return 1;
+        href: '/favicon.ico',
-    return b.size - a.size;
+        hasSizes: false
-  });
-
-  const timeoutMs = 2000;
-
-  for (const icon of icons) {
-    try {
-      const controller = new AbortController();
-      const timeout = setTimeout(() => controller.abort(), timeoutMs);
-
-      const resp = await fetch(icon.href, {
-        signal: controller.signal,
-        redirect: 'follow'
       });
+    }
 
-      clearTimeout(timeout);
+    // sort preference: highest resolution first, then apple-touch-icon, then regular icons
+    icons.sort((a, b) => {
+      // First priority: actual size (highest first)
+      if (a.size !== b.size) {
+        return b.size - a.size;
+      }
+      
+      // Second priority: apple-touch-icon over regular icon
+      const isAppleA = /apple-touch-icon/.test(a.rel);
+      const isAppleB = /apple-touch-icon/.test(b.rel);
+      if (isAppleA && !isAppleB) return -1;
+      if (!isAppleA && isAppleB) return 1;
+      
+      // Third priority: icons with no size attribute (fallback icons) last
+      const hasNoSizeA = !a.hasSizes;
+      const hasNoSizeB = !b.hasSizes;
+      if (hasNoSizeA && !hasNoSizeB) return 1;
+      if (!hasNoSizeA && hasNoSizeB) return -1;
+      
+      return 0;
+    });
 
-      if (!resp.ok) {
+    const timeoutMs = 2000;
+
+    for (const icon of icons) {
+      try {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+        const resp = await fetch(icon.href, {
+          signal: controller.signal,
+          redirect: 'follow'
+        });
+
+        clearTimeout(timeout);
+
+        if (!resp.ok) {
+          continue;
+        }
+
+        const blob = await resp.blob();
+
+        // Convert blob to base64
+        const reader = new FileReader();
+        return await new Promise(resolve => {
+          reader.onloadend = () => {
+            resolve({
+              url: icon.href,
+              base64: reader.result.split(",")[1]
+            });
+          };
+          reader.readAsDataURL(blob);
+        });
+
+      } catch (e) {
         continue;
       }
-
-      const blob = await resp.blob();
-
-      // Convert blob to base64
-      const reader = new FileReader();
-      return await new Promise(resolve => {
-        reader.onloadend = () => {
-          resolve({
-            url: icon.href,
-            base64: reader.result.split(",")[1]
-          });
-        };
-        reader.readAsDataURL(blob);
-      });
-
-    } catch (e) {
-      continue;
     }
-  }
 
-  // nothing found
+    // nothing found
-  return null;
+    return null;
+  };
+
+  // Auto-execute and return result for page.evaluate()
+  return await window.getFaviconAsBlob();
 })();
+

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -47,6 +47,7 @@ async () => {
             'nicht lieferbar',
             'nicht verfügbar',
             'nicht vorrätig',
+            'nicht mehr lieferbar',
             'nicht zur verfügung',
             'nie znaleziono produktów',
             'niet beschikbaar',

changedetectionio/content_fetchers/webdriver_selenium.py

@@ -4,9 +4,10 @@ import time
 from loguru import logger
 from changedetectionio.content_fetchers.base import Fetcher
 
+
 class fetcher(Fetcher):
     if os.getenv("WEBDRIVER_URL"):
-        fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL"))
+        fetcher_description = f"WebDriver Chrome/Javascript via \"{os.getenv('WEBDRIVER_URL', '')}\""
     else:
         fetcher_description = "WebDriver Chrome/Javascript"
 
@@ -25,7 +26,6 @@ class fetcher(Fetcher):
             self.browser_connection_is_custom = True
             self.browser_connection_url = custom_browser_connection_url
 
-
         ##### PROXY SETUP #####
 
         proxy_sources = [
@@ -38,7 +38,7 @@ class fetcher(Fetcher):
             os.getenv('webdriver_proxyHttps'),
             os.getenv('webdriver_httpsProxy'),
             os.getenv('webdriver_sslProxy'),
-            proxy_override, # last one should override
+            proxy_override,  # last one should override
         ]
         # The built in selenium proxy handling is super unreliable!!! so we just grab which ever proxy setting we can find and throw it in --proxy-server=
         for k in filter(None, proxy_sources):
@@ -46,20 +46,21 @@ class fetcher(Fetcher):
                 continue
             self.proxy_url = k.strip()
 
-
     async def run(self,
-            url,
+                  fetch_favicon=True,
-            timeout,
+                  current_include_filters=None,
-            request_headers,
+                  empty_pages_are_a_change=False,
-            request_body,
+                  ignore_status_codes=False,
-            request_method,
+                  is_binary=False,
-            ignore_status_codes=False,
+                  request_body=None,
-            current_include_filters=None,
+                  request_headers=None,
-            is_binary=False,
+                  request_method=None,
-            empty_pages_are_a_change=False):
+                  timeout=None,
+                  url=None,
+                  ):
 
         import asyncio
-        
+
         # Wrap the entire selenium operation in a thread executor
         def _run_sync():
             from selenium.webdriver.chrome.options import Options as ChromeOptions
@@ -140,4 +141,3 @@ class fetcher(Fetcher):
         # Run the selenium operations in a thread pool to avoid blocking the event loop
         loop = asyncio.get_event_loop()
         await loop.run_in_executor(None, _run_sync)
-

changedetectionio/flask_app.py

@@ -12,7 +12,7 @@ from blinker import signal
 
 from changedetectionio.strtobool import strtobool
 from threading import Event
-from changedetectionio.custom_queue import SignalPriorityQueue, AsyncSignalPriorityQueue, NotificationQueue
+from changedetectionio.queue_handlers import RecheckPriorityQueue, NotificationQueue
 from changedetectionio import worker_handler
 
 from flask import (
@@ -48,8 +48,8 @@ datastore = None
 ticker_thread = None
 extra_stylesheets = []
 
-# Use async queue by default, keep sync for backward compatibility  
+# Use bulletproof janus-based queues for sync/async reliability  
-update_q = AsyncSignalPriorityQueue() if worker_handler.USE_ASYNC_WORKERS else SignalPriorityQueue()
+update_q = RecheckPriorityQueue()
 notification_q = NotificationQueue()
 MAX_QUEUE_SIZE = 2000
 
@@ -329,7 +329,7 @@ def changedetection_app(config=None, datastore_o=None):
                            resource_class_kwargs={'datastore': datastore})
 
     watch_api.add_resource(Tag, '/api/v1/tag', '/api/v1/tag/<string:uuid>',
-                           resource_class_kwargs={'datastore': datastore})
+                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
                            
     watch_api.add_resource(Search, '/api/v1/search',
                            resource_class_kwargs={'datastore': datastore})
@@ -844,16 +844,22 @@ def ticker_thread_check_time_launch_checks():
 
                     # Use Epoch time as priority, so we get a "sorted" PriorityQueue, but we can still push a priority 1 into it.
                     priority = int(time.time())
-                    logger.debug(
-                        f"> Queued watch UUID {uuid} "
-                        f"last checked at {watch['last_checked']} "
-                        f"queued at {now:0.2f} priority {priority} "
-                        f"jitter {watch.jitter_seconds:0.2f}s, "
-                        f"{now - watch['last_checked']:0.2f}s since last checked")
 
                     # Into the queue with you
-                    worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=priority, item={'uuid': uuid}))
+                    queued_successfully = worker_handler.queue_item_async_safe(update_q,
-
+                                                                               queuedWatchMetaData.PrioritizedItem(priority=priority,
+                                                                                                                   item={'uuid': uuid})
+                                                                               )
+                    if queued_successfully:
+                        logger.debug(
+                            f"> Queued watch UUID {uuid} "
+                            f"last checked at {watch['last_checked']} "
+                            f"queued at {now:0.2f} priority {priority} "
+                            f"jitter {watch.jitter_seconds:0.2f}s, "
+                            f"{now - watch['last_checked']:0.2f}s since last checked")
+                    else:
+                        logger.critical(f"CRITICAL: Failed to queue watch UUID {uuid} in ticker thread!")
+                        
                     # Reset for next time
                     watch.jitter_seconds = 0
 

changedetectionio/forms.py

@@ -23,11 +23,14 @@ from wtforms import (
 )
 from flask_wtf.file import FileField, FileAllowed
 from wtforms.fields import FieldList
+from wtforms.utils import unset_value
 
 from wtforms.validators import ValidationError
 
 from validators.url import url as url_validator
 
+from changedetectionio.widgets import TernaryNoneBooleanField
+
 
 # default
 # each select <option data-enabled="enabled-0-0"
@@ -54,6 +57,8 @@ valid_method = {
 
 default_method = 'GET'
 allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
+REQUIRE_ATLEAST_ONE_TIME_PART_MESSAGE_DEFAULT='At least one time interval (weeks, days, hours, minutes, or seconds) must be specified.'
+REQUIRE_ATLEAST_ONE_TIME_PART_WHEN_NOT_GLOBAL_DEFAULT='At least one time interval (weeks, days, hours, minutes, or seconds) must be specified when not using global settings.'
 
 class StringListField(StringField):
     widget = widgets.TextArea()
@@ -210,6 +215,33 @@ class ScheduleLimitForm(Form):
         self.sunday.form.enabled.label.text = "Sunday"
 
 
+def validate_time_between_check_has_values(form):
+    """
+    Custom validation function for TimeBetweenCheckForm.
+    Returns True if at least one time interval field has a value > 0.
+    """
+    return any([
+        form.weeks.data and form.weeks.data > 0,
+        form.days.data and form.days.data > 0,
+        form.hours.data and form.hours.data > 0,
+        form.minutes.data and form.minutes.data > 0,
+        form.seconds.data and form.seconds.data > 0
+    ])
+
+
+class RequiredTimeInterval(object):
+    """
+    WTForms validator that ensures at least one time interval field has a value > 0.
+    Use this with FormField(TimeBetweenCheckForm, validators=[RequiredTimeInterval()]).
+    """
+    def __init__(self, message=None):
+        self.message = message or 'At least one time interval (weeks, days, hours, minutes, or seconds) must be specified.'
+
+    def __call__(self, form, field):
+        if not validate_time_between_check_has_values(field.form):
+            raise ValidationError(self.message)
+
+
 class TimeBetweenCheckForm(Form):
     weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
     days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
@@ -218,6 +250,123 @@ class TimeBetweenCheckForm(Form):
     seconds = IntegerField('Seconds', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
     # @todo add total seconds minimum validatior = minimum_seconds_recheck_time
 
+    def __init__(self, formdata=None, obj=None, prefix="", data=None, meta=None, **kwargs):
+        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
+        self.require_at_least_one = kwargs.get('require_at_least_one', False)
+        self.require_at_least_one_message = kwargs.get('require_at_least_one_message', REQUIRE_ATLEAST_ONE_TIME_PART_MESSAGE_DEFAULT)
+
+    def validate(self, **kwargs):
+        """Custom validation that can optionally require at least one time interval."""
+        # Run normal field validation first
+        if not super().validate(**kwargs):
+            return False
+
+        # Apply optional "at least one" validation
+        if self.require_at_least_one:
+            if not validate_time_between_check_has_values(self):
+                # Add error to the form's general errors (not field-specific)
+                if not hasattr(self, '_formdata_errors'):
+                    self._formdata_errors = []
+                self._formdata_errors.append(self.require_at_least_one_message)
+                return False
+
+        return True
+
+
+class EnhancedFormField(FormField):
+    """
+    An enhanced FormField that supports conditional validation with top-level error messages.
+    Adds a 'top_errors' property for validation errors at the FormField level.
+    """
+
+    def __init__(self, form_class, label=None, validators=None, separator="-",
+                 conditional_field=None, conditional_message=None, conditional_test_function=None, **kwargs):
+        """
+        Initialize EnhancedFormField with optional conditional validation.
+
+        :param conditional_field: Name of the field this FormField depends on (e.g. 'time_between_check_use_default')
+        :param conditional_message: Error message to show when validation fails
+        :param conditional_test_function: Custom function to test if FormField has valid values.
+                                        Should take self.form as parameter and return True if valid.
+        """
+        super().__init__(form_class, label, validators, separator, **kwargs)
+        self.top_errors = []
+        self.conditional_field = conditional_field
+        self.conditional_message = conditional_message or "At least one field must have a value when not using defaults."
+        self.conditional_test_function = conditional_test_function
+
+    def validate(self, form, extra_validators=()):
+        """
+        Custom validation that supports conditional logic and stores top-level errors.
+        """
+        self.top_errors = []
+
+        # First run the normal FormField validation
+        base_valid = super().validate(form, extra_validators)
+
+        # Apply conditional validation if configured
+        if self.conditional_field and hasattr(form, self.conditional_field):
+            conditional_field_obj = getattr(form, self.conditional_field)
+
+            # If the conditional field is False/unchecked, check if this FormField has any values
+            if not conditional_field_obj.data:
+                # Use custom test function if provided, otherwise use generic fallback
+                if self.conditional_test_function:
+                    has_any_value = self.conditional_test_function(self.form)
+                else:
+                    # Generic fallback - check if any field has truthy data
+                    has_any_value = any(field.data for field in self.form if hasattr(field, 'data') and field.data)
+
+                if not has_any_value:
+                    self.top_errors.append(self.conditional_message)
+                    base_valid = False
+
+        return base_valid
+
+
+class RequiredFormField(FormField):
+    """
+    A FormField that passes require_at_least_one=True to TimeBetweenCheckForm.
+    Use this when you want the sub-form to always require at least one value.
+    """
+
+    def __init__(self, form_class, label=None, validators=None, separator="-", **kwargs):
+        super().__init__(form_class, label, validators, separator, **kwargs)
+
+    def process(self, formdata, data=unset_value, extra_filters=None):
+        if extra_filters:
+            raise TypeError(
+                "FormField cannot take filters, as the encapsulated"
+                "data is not mutable."
+            )
+
+        if data is unset_value:
+            try:
+                data = self.default()
+            except TypeError:
+                data = self.default
+            self._obj = data
+
+        self.object_data = data
+
+        prefix = self.name + self.separator
+        # Pass require_at_least_one=True to the sub-form
+        if isinstance(data, dict):
+            self.form = self.form_class(formdata=formdata, prefix=prefix, require_at_least_one=True, **data)
+        else:
+            self.form = self.form_class(formdata=formdata, obj=data, prefix=prefix, require_at_least_one=True)
+
+    @property
+    def errors(self):
+        """Include sub-form validation errors"""
+        form_errors = self.form.errors
+        # Add any general form errors to a special 'form' key
+        if hasattr(self.form, '_formdata_errors') and self.form._formdata_errors:
+            form_errors = dict(form_errors)  # Make a copy
+            form_errors['form'] = self.form._formdata_errors
+        return form_errors
+
+
 # Separated by  key:value
 class StringDictKeyValue(StringField):
     widget = widgets.TextArea()
@@ -346,7 +495,7 @@ class ValidateJinja2Template(object):
         joined_data = ' '.join(map(str, field.data)) if isinstance(field.data, list) else f"{field.data}"
 
         try:
-            jinja2_env = ImmutableSandboxedEnvironment(loader=BaseLoader)
+            jinja2_env = ImmutableSandboxedEnvironment(loader=BaseLoader, extensions=['jinja2_time.TimeExtension'])
             jinja2_env.globals.update(notification.valid_tokens)
             # Extra validation tokens provided on the form_class(... extra_tokens={}) setup
             if hasattr(field, 'extra_notification_tokens'):
@@ -548,7 +697,6 @@ class commonSettingsForm(Form):
         self.notification_title.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
         self.notification_urls.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
 
-    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
     fetch_backend = RadioField(u'Fetch Method', choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_format = SelectField('Notification format', choices=valid_notification_formats.keys())
@@ -582,11 +730,16 @@ class processor_text_json_diff_form(commonSettingsForm):
     url = fields.URLField('URL', validators=[validateURL()])
     tags = StringTagUUID('Group tag', [validators.Optional()], default='')
 
-    time_between_check = FormField(TimeBetweenCheckForm)
+    time_between_check = EnhancedFormField(
+        TimeBetweenCheckForm,
+        conditional_field='time_between_check_use_default',
+        conditional_message=REQUIRE_ATLEAST_ONE_TIME_PART_WHEN_NOT_GLOBAL_DEFAULT,
+        conditional_test_function=validate_time_between_check_has_values
+    )
 
     time_schedule_limit = FormField(ScheduleLimitForm)
 
-    time_between_check_use_default = BooleanField('Use global settings for time between check', default=False)
+    time_between_check_use_default = BooleanField('Use global settings for time between check and scheduler.', default=False)
 
     include_filters = StringListField('CSS/JSONPath/JQ/XPath Filters', [ValidateCSSJSONXPATHInput()], default='')
 
@@ -616,18 +769,18 @@ class processor_text_json_diff_form(commonSettingsForm):
     text_should_not_be_present = StringListField('Block change-detection while text matches', [validators.Optional(), ValidateListRegex()])
     webdriver_js_execute_code = TextAreaField('Execute JavaScript before change detection', render_kw={"rows": "5"}, validators=[validators.Optional()])
 
-    save_button = SubmitField('Save', render_kw={"class": "pure-button button-small pure-button-primary"})
+    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
 
     proxy = RadioField('Proxy')
+    # filter_failure_notification_send @todo make ternary
     filter_failure_notification_send = BooleanField(
         'Send a notification when the filter can no longer be found on the page', default=False)
-
+    notification_muted = TernaryNoneBooleanField('Notifications', default=None, yes_text="Muted", no_text="On")
-    notification_muted = BooleanField('Notifications Muted / Off', default=False)
     notification_screenshot = BooleanField('Attach screenshot to notification (where possible)', default=False)
 
     conditions_match_logic = RadioField(u'Match', choices=[('ALL', 'Match all of the following'),('ANY', 'Match any of the following')], default='ALL')
     conditions = FieldList(FormField(ConditionFormRow), min_entries=1)  # Add rule logic here
-
+    use_page_title_in_list = TernaryNoneBooleanField('Use page <title> in list', default=None)
 
     def extra_tab_content(self):
         return None
@@ -727,7 +880,7 @@ class DefaultUAInputForm(Form):
 
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
-    time_between_check = FormField(TimeBetweenCheckForm)
+    time_between_check = RequiredFormField(TimeBetweenCheckForm)
     time_schedule_limit = FormField(ScheduleLimitForm)
     proxy = RadioField('Proxy')
     jitter_seconds = IntegerField('Random jitter seconds ± check',
@@ -755,6 +908,7 @@ class globalSettingsApplicationUIForm(Form):
     open_diff_in_new_tab = BooleanField("Open 'History' page in a new tab", default=True, validators=[validators.Optional()])
     socket_io_enabled = BooleanField('Realtime UI Updates Enabled', default=True, validators=[validators.Optional()])
     favicons_enabled = BooleanField('Favicons Enabled', default=True, validators=[validators.Optional()])
+    use_page_title_in_list = BooleanField('Use page <title> in watch overview list') #BooleanField=True
 
 # datastore.data['settings']['application']..
 class globalSettingsApplicationForm(commonSettingsForm):
@@ -779,7 +933,7 @@ class globalSettingsApplicationForm(commonSettingsForm):
 
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
-    shared_diff_access = BooleanField('Allow access to view diff page when password is enabled', default=False, validators=[validators.Optional()])
+    shared_diff_access = BooleanField('Allow anonymous access to watch history page when password is enabled', default=False, validators=[validators.Optional()])
     rss_hide_muted_watches = BooleanField('Hide muted watches from RSS feed', default=True,
                                       validators=[validators.Optional()])
     filter_failure_notification_threshold_attempts = IntegerField('Number of times the filter can be missing before sending a notification',
@@ -801,7 +955,7 @@ class globalSettingsForm(Form):
 
     requests = FormField(globalSettingsRequestForm)
     application = FormField(globalSettingsApplicationForm)
-    save_button = SubmitField('Save', render_kw={"class": "pure-button button-small pure-button-primary"})
+    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
 
 
 class extractDataForm(Form):

changedetectionio/html_tools.py

@@ -1,6 +1,7 @@
 from loguru import logger
 from lxml import etree
 from typing import List
+import html
 import json
 import re
 
@@ -9,6 +10,11 @@ TEXT_FILTER_LIST_LINE_SUFFIX = "<br>"
 TRANSLATE_WHITESPACE_TABLE = str.maketrans('', '', '\r\n\t ')
 PERL_STYLE_REGEX = r'^/(.*?)/([a-z]*)?$'
 
+TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.I | re.S)
+META_CS  = re.compile(r'<meta[^>]+charset=["\']?\s*([a-z0-9_\-:+.]+)', re.I)
+META_CT  = re.compile(r'<meta[^>]+http-equiv=["\']?content-type["\']?[^>]*content=["\'][^>]*charset=([a-z0-9_\-:+.]+)', re.I)
+
+
 # 'price' , 'lowPrice', 'highPrice' are usually under here
 # All of those may or may not appear on different websites - I didnt find a way todo case-insensitive searching here
 LD_JSON_PRODUCT_OFFER_SELECTORS = ["json:$..offers", "json:$..Offers"]
@@ -510,3 +516,43 @@ def get_triggered_text(content, trigger_text):
         i += 1
 
     return triggered_text
+
+
+def extract_title(data: bytes | str, sniff_bytes: int = 2048, scan_chars: int = 8192) -> str | None:
+    try:
+        # Only decode/process the prefix we need for title extraction
+        match data:
+            case bytes() if data.startswith((b"\xff\xfe", b"\xfe\xff")):
+                prefix = data[:scan_chars * 2].decode("utf-16", errors="replace")
+            case bytes() if data.startswith((b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff")):
+                prefix = data[:scan_chars * 4].decode("utf-32", errors="replace")
+            case bytes():
+                try:
+                    prefix = data[:scan_chars].decode("utf-8")
+                except UnicodeDecodeError:
+                    try:
+                        head = data[:sniff_bytes].decode("ascii", errors="ignore")
+                        if m := (META_CS.search(head) or META_CT.search(head)):
+                            enc = m.group(1).lower()
+                        else:
+                            enc = "cp1252"
+                        prefix = data[:scan_chars * 2].decode(enc, errors="replace")
+                    except Exception as e:
+                        logger.error(f"Title extraction encoding detection failed: {e}")
+                        return None
+            case str():
+                prefix = data[:scan_chars] if len(data) > scan_chars else data
+            case _:
+                logger.error(f"Title extraction received unsupported data type: {type(data)}")
+                return None
+
+        # Search only in the prefix
+        if m := TITLE_RE.search(prefix):
+            title = html.unescape(" ".join(m.group(1).split())).strip()
+            # Some safe limit
+            return title[:2000]
+        return None
+        
+    except Exception as e:
+        logger.error(f"Title extraction failed: {e}")
+        return None

changedetectionio/model/App.py

@@ -39,12 +39,12 @@ class model(dict):
                     'api_access_token_enabled': True,
                     'base_url' : None,
                     'empty_pages_are_a_change': False,
-                    'extract_title_as_title': False,
                     'fetch_backend': getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
                     'filter_failure_notification_threshold_attempts': _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT,
                     'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
                     'global_subtractive_selectors': [],
                     'ignore_whitespace': True,
+                    'ignore_status_codes': False, #@todo implement, as ternary.
                     'notification_body': default_notification_body,
                     'notification_format': default_notification_format,
                     'notification_title': default_notification_title,
@@ -57,10 +57,11 @@ class model(dict):
                     'rss_hide_muted_watches': True,
                     'schema_version' : 0,
                     'shared_diff_access': False,
-                    'webdriver_delay': None , # Extra delay in seconds before extracting text
                     'tags': {}, #@todo use Tag.model initialisers
                     'timezone': None, # Default IANA timezone name
+                    'webdriver_delay': None , # Extra delay in seconds before extracting text
                     'ui': {
+                        'use_page_title_in_list': True,
                         'open_diff_in_new_tab': True,
                         'socket_io_enabled': True,
                         'favicons_enabled': True

changedetectionio/model/Watch.py

@@ -14,6 +14,8 @@ from ..html_tools import TRANSLATE_WHITESPACE_TABLE
 # Allowable protocols, protects against javascript: etc
 # file:// is further checked by ALLOW_FILE_URI
 SAFE_PROTOCOL_REGEX='^(http|https|ftp|file):'
+FAVICON_RESAVE_THRESHOLD_SECONDS=86400
+
 
 minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3))
 mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
@@ -167,8 +169,8 @@ class model(watch_base):
 
     @property
     def label(self):
-        # Used for sorting
+        # Used for sorting, display, etc
-        return self.get('title') if self.get('title') else self.get('url')
+        return self.get('title') or self.get('page_title') or self.get('url')
 
     @property
     def last_changed(self):
@@ -420,6 +422,28 @@ class model(watch_base):
         # False is not an option for AppRise, must be type None
         return None
 
+    def favicon_is_expired(self):
+        favicon_fname = self.get_favicon_filename()
+        import glob
+        import time
+
+        if not favicon_fname:
+            return True
+        try:
+            fname = next(iter(glob.glob(os.path.join(self.watch_data_dir, "favicon.*"))), None)
+            logger.trace(f"Favicon file maybe found at {fname}")
+            if os.path.isfile(fname):
+                file_age = int(time.time() - os.path.getmtime(fname))
+                logger.trace(f"Favicon file age is {file_age}s")
+                if file_age < FAVICON_RESAVE_THRESHOLD_SECONDS:
+                    return False
+        except Exception as e:
+            logger.critical(f"Exception checking Favicon age {str(e)}")
+            return True
+
+        # Also in the case that the file didnt exist
+        return True
+
     def bump_favicon(self, url, favicon_base_64: str) -> None:
         from urllib.parse import urlparse
         import base64

changedetectionio/model/__init__.py

@@ -24,7 +24,6 @@ class watch_base(dict):
             'content-type': None,
             'date_created': None,
             'extract_text': [],  # Extract text by regex after filters
-            'extract_title_as_title': False,
             'fetch_backend': 'system',  # plaintext, playwright etc
             'fetch_time': 0.0,
             'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
@@ -35,6 +34,7 @@ class watch_base(dict):
             'has_ldjson_price_data': None,
             'headers': {},  # Extra headers to send
             'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
+            'ignore_status_codes': None,
             'in_stock_only': True,  # Only trigger change on going to instock from out-of-stock
             'include_filters': [],
             'last_checked': 0,
@@ -49,6 +49,7 @@ class watch_base(dict):
             'notification_screenshot': False,  # Include the latest screenshot if available and supported by the apprise URL
             'notification_title': None,
             'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
+            'page_title': None, # <title> from the page
             'paused': False,
             'previous_md5': False,
             'previous_md5_before_filters': False,  # Used for skipping changedetection entirely
@@ -122,12 +123,13 @@ class watch_base(dict):
                     }
                 },
             },
-            'title': None,
+            'title': None, # An arbitrary field that overrides 'page_title'
             'track_ldjson_price_data': None,
             'trim_text_whitespace': False,
             'remove_duplicate_lines': False,
             'trigger_text': [],  # List of text or regex to wait for until a change is detected
             'url': '',
+            'use_page_title_in_list': None, # None = use system settings
             'uuid': str(uuid.uuid4()),
             'webdriver_delay': None,
             'webdriver_js_execute_code': None,  # Run before change-detection

changedetectionio/notification/handler.py

@@ -149,7 +149,7 @@ def create_notification_parameters(n_object, datastore):
     uuid = n_object['uuid'] if 'uuid' in n_object else ''
 
     if uuid:
-        watch_title = datastore.data['watching'][uuid].get('title', '')
+        watch_title = datastore.data['watching'][uuid].label
         tag_list = []
         tags = datastore.get_all_tags_for_watch(uuid)
         if tags:

changedetectionio/processors/__init__.py

@@ -146,18 +146,19 @@ class difference_detection_processor():
 
         # And here we go! call the right browser with browser-specific settings
         empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
-
         # All fetchers are now async
-        await self.fetcher.run(url=url,
+        await self.fetcher.run(
-                               timeout=timeout,
+            current_include_filters=self.watch.get('include_filters'),
-                               request_headers=request_headers,
+            empty_pages_are_a_change=empty_pages_are_a_change,
-                               request_body=request_body,
+            fetch_favicon=self.watch.favicon_is_expired(),
-                               request_method=request_method,
+            ignore_status_codes=ignore_status_codes,
-                               ignore_status_codes=ignore_status_codes,
+            is_binary=is_binary,
-                               current_include_filters=self.watch.get('include_filters'),
+            request_body=request_body,
-                               is_binary=is_binary,
+            request_headers=request_headers,
-                               empty_pages_are_a_change=empty_pages_are_a_change
+            request_method=request_method,
-                               )
+            timeout=timeout,
+            url=url,
+       )
 
         #@todo .quit here could go on close object, so we can run JS if change-detected
         self.fetcher.quit(watch=self.watch)

changedetectionio/processors/text_json_diff/processor.py

@@ -251,8 +251,7 @@ class perform_site_check(difference_detection_processor):
         update_obj["last_check_status"] = self.fetcher.get_last_status_code()
 
         # 615 Extract text by regex
-        extract_text = watch.get('extract_text', [])
+        extract_text = list(dict.fromkeys(watch.get('extract_text', []) + self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='extract_text')))
-        extract_text += self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='extract_text')
         if len(extract_text) > 0:
             regex_matched_output = []
             for s_re in extract_text:
@@ -311,8 +310,7 @@ class perform_site_check(difference_detection_processor):
 
         ############ Blocking rules, after checksum #################
         blocked = False
-        trigger_text = watch.get('trigger_text', [])
+        trigger_text = list(dict.fromkeys(watch.get('trigger_text', []) + self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='trigger_text')))
-        trigger_text += self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='trigger_text')
         if len(trigger_text):
             # Assume blocked
             blocked = True
@@ -326,8 +324,7 @@ class perform_site_check(difference_detection_processor):
             if result:
                 blocked = False
 
-        text_should_not_be_present = watch.get('text_should_not_be_present', [])
+        text_should_not_be_present = list(dict.fromkeys(watch.get('text_should_not_be_present', []) + self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='text_should_not_be_present')))
-        text_should_not_be_present += self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='text_should_not_be_present')
         if len(text_should_not_be_present):
             # If anything matched, then we should block a change from happening
             result = html_tools.strip_ignore_text(content=str(stripped_text_from_html),

changedetectionio/queue_handlers.py

@@ -0,0 +1,435 @@
+from blinker import signal
+from loguru import logger
+from typing import Dict, List, Any, Optional
+import heapq
+import queue
+import threading
+
+try:
+    import janus
+except ImportError:
+    logger.critical(f"CRITICAL: janus library is required. Install with: pip install janus")
+    raise
+
+
+class RecheckPriorityQueue:
+    """
+    Ultra-reliable priority queue using janus for async/sync bridging.
+    
+    CRITICAL DESIGN NOTE: Both sync_q and async_q are required because:
+    - sync_q: Used by Flask routes, ticker threads, and other synchronous code
+    - async_q: Used by async workers (the actual fetchers/processors) and coroutines
+    
+    DO NOT REMOVE EITHER INTERFACE - they bridge different execution contexts:
+    - Synchronous code (Flask, threads) cannot use async methods without blocking
+    - Async code cannot use sync methods without blocking the event loop
+    - janus provides the only safe bridge between these two worlds
+    
+    Attempting to unify to async-only would require:
+    - Converting all Flask routes to async (major breaking change)
+    - Using asyncio.run() in sync contexts (causes deadlocks)
+    - Thread-pool wrapping (adds complexity and overhead)
+    
+    Minimal implementation focused on reliability:
+    - Pure janus for sync/async bridge
+    - Thread-safe priority ordering  
+    - Bulletproof error handling with critical logging
+    """
+    
+    def __init__(self, maxsize: int = 0):
+        try:
+            self._janus_queue = janus.Queue(maxsize=maxsize)
+            # BOTH interfaces required - see class docstring for why
+            self.sync_q = self._janus_queue.sync_q   # Flask routes, ticker thread
+            self.async_q = self._janus_queue.async_q # Async workers
+            
+            # Priority storage - thread-safe
+            self._priority_items = []
+            self._lock = threading.RLock()
+            
+            # Signals for UI updates
+            self.queue_length_signal = signal('queue_length')
+            
+            logger.debug("RecheckPriorityQueue initialized successfully")
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to initialize RecheckPriorityQueue: {str(e)}")
+            raise
+    
+    # SYNC INTERFACE (for ticker thread)
+    def put(self, item, block: bool = True, timeout: Optional[float] = None):
+        """Thread-safe sync put with priority ordering"""
+        try:
+            # Add to priority storage
+            with self._lock:
+                heapq.heappush(self._priority_items, item)
+            
+            # Notify via janus sync queue
+            self.sync_q.put(True, block=block, timeout=timeout)
+            
+            # Emit signals
+            self._emit_put_signals(item)
+            
+            logger.debug(f"Successfully queued item: {self._get_item_uuid(item)}")
+            return True
+            
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to put item {self._get_item_uuid(item)}: {str(e)}")
+            # Remove from priority storage if janus put failed
+            try:
+                with self._lock:
+                    if item in self._priority_items:
+                        self._priority_items.remove(item)
+                        heapq.heapify(self._priority_items)
+            except Exception as cleanup_e:
+                logger.critical(f"CRITICAL: Failed to cleanup after put failure: {str(e)}")
+            return False
+    
+    def get(self, block: bool = True, timeout: Optional[float] = None):
+        """Thread-safe sync get with priority ordering"""
+        try:
+            # Wait for notification
+            self.sync_q.get(block=block, timeout=timeout)
+            
+            # Get highest priority item
+            with self._lock:
+                if not self._priority_items:
+                    logger.critical(f"CRITICAL: Queue notification received but no priority items available")
+                    raise Exception("Priority queue inconsistency")
+                item = heapq.heappop(self._priority_items)
+            
+            # Emit signals
+            self._emit_get_signals()
+            
+            logger.debug(f"Successfully retrieved item: {self._get_item_uuid(item)}")
+            return item
+            
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to get item from queue: {str(e)}")
+            raise
+    
+    # ASYNC INTERFACE (for workers)
+    async def async_put(self, item):
+        """Pure async put with priority ordering"""
+        try:
+            # Add to priority storage
+            with self._lock:
+                heapq.heappush(self._priority_items, item)
+            
+            # Notify via janus async queue
+            await self.async_q.put(True)
+            
+            # Emit signals
+            self._emit_put_signals(item)
+            
+            logger.debug(f"Successfully async queued item: {self._get_item_uuid(item)}")
+            return True
+            
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to async put item {self._get_item_uuid(item)}: {str(e)}")
+            # Remove from priority storage if janus put failed
+            try:
+                with self._lock:
+                    if item in self._priority_items:
+                        self._priority_items.remove(item)
+                        heapq.heapify(self._priority_items)
+            except Exception as cleanup_e:
+                logger.critical(f"CRITICAL: Failed to cleanup after async put failure: {str(e)}")
+            return False
+    
+    async def async_get(self):
+        """Pure async get with priority ordering"""
+        try:
+            # Wait for notification
+            await self.async_q.get()
+            
+            # Get highest priority item
+            with self._lock:
+                if not self._priority_items:
+                    logger.critical(f"CRITICAL: Async queue notification received but no priority items available")
+                    raise Exception("Priority queue inconsistency")
+                item = heapq.heappop(self._priority_items)
+            
+            # Emit signals
+            self._emit_get_signals()
+            
+            logger.debug(f"Successfully async retrieved item: {self._get_item_uuid(item)}")
+            return item
+            
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to async get item from queue: {str(e)}")
+            raise
+    
+    # UTILITY METHODS
+    def qsize(self) -> int:
+        """Get current queue size"""
+        try:
+            with self._lock:
+                return len(self._priority_items)
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to get queue size: {str(e)}")
+            return 0
+    
+    def empty(self) -> bool:
+        """Check if queue is empty"""
+        return self.qsize() == 0
+    
+    def close(self):
+        """Close the janus queue"""
+        try:
+            self._janus_queue.close()
+            logger.debug("RecheckPriorityQueue closed successfully")
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to close RecheckPriorityQueue: {str(e)}")
+    
+    # COMPATIBILITY METHODS (from original implementation)
+    @property
+    def queue(self):
+        """Provide compatibility with original queue access"""
+        try:
+            with self._lock:
+                return list(self._priority_items)
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to get queue list: {str(e)}")
+            return []
+    
+    def get_uuid_position(self, target_uuid: str) -> Dict[str, Any]:
+        """Find position of UUID in queue"""
+        try:
+            with self._lock:
+                queue_list = list(self._priority_items)
+                total_items = len(queue_list)
+                
+                if total_items == 0:
+                    return {'position': None, 'total_items': 0, 'priority': None, 'found': False}
+                
+                # Find target item
+                for item in queue_list:
+                    if (hasattr(item, 'item') and isinstance(item.item, dict) and 
+                        item.item.get('uuid') == target_uuid):
+                        
+                        # Count items with higher priority
+                        position = sum(1 for other in queue_list if other.priority < item.priority)
+                        return {
+                            'position': position,
+                            'total_items': total_items, 
+                            'priority': item.priority,
+                            'found': True
+                        }
+                
+                return {'position': None, 'total_items': total_items, 'priority': None, 'found': False}
+                
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to get UUID position for {target_uuid}: {str(e)}")
+            return {'position': None, 'total_items': 0, 'priority': None, 'found': False}
+    
+    def get_all_queued_uuids(self, limit: Optional[int] = None, offset: int = 0) -> Dict[str, Any]:
+        """Get all queued UUIDs with pagination"""
+        try:
+            with self._lock:
+                queue_list = sorted(self._priority_items)  # Sort by priority
+                total_items = len(queue_list)
+                
+                if total_items == 0:
+                    return {'items': [], 'total_items': 0, 'returned_items': 0, 'has_more': False}
+                
+                # Apply pagination
+                end_idx = min(offset + limit, total_items) if limit else total_items
+                items_to_process = queue_list[offset:end_idx]
+                
+                result = []
+                for position, item in enumerate(items_to_process, start=offset):
+                    if (hasattr(item, 'item') and isinstance(item.item, dict) and 
+                        'uuid' in item.item):
+                        result.append({
+                            'uuid': item.item['uuid'],
+                            'position': position,
+                            'priority': item.priority
+                        })
+                
+                return {
+                    'items': result,
+                    'total_items': total_items,
+                    'returned_items': len(result),
+                    'has_more': (offset + len(result)) < total_items
+                }
+                
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to get all queued UUIDs: {str(e)}")
+            return {'items': [], 'total_items': 0, 'returned_items': 0, 'has_more': False}
+    
+    def get_queue_summary(self) -> Dict[str, Any]:
+        """Get queue summary statistics"""
+        try:
+            with self._lock:
+                queue_list = list(self._priority_items)
+                total_items = len(queue_list)
+                
+                if total_items == 0:
+                    return {
+                        'total_items': 0, 'priority_breakdown': {},
+                        'immediate_items': 0, 'clone_items': 0, 'scheduled_items': 0
+                    }
+                
+                immediate_items = clone_items = scheduled_items = 0
+                priority_counts = {}
+                
+                for item in queue_list:
+                    priority = item.priority
+                    priority_counts[priority] = priority_counts.get(priority, 0) + 1
+                    
+                    if priority == 1:
+                        immediate_items += 1
+                    elif priority == 5:
+                        clone_items += 1
+                    elif priority > 100:
+                        scheduled_items += 1
+                
+                return {
+                    'total_items': total_items,
+                    'priority_breakdown': priority_counts,
+                    'immediate_items': immediate_items,
+                    'clone_items': clone_items,
+                    'scheduled_items': scheduled_items,
+                    'min_priority': min(priority_counts.keys()) if priority_counts else None,
+                    'max_priority': max(priority_counts.keys()) if priority_counts else None
+                }
+                
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to get queue summary: {str(e)}")
+            return {'total_items': 0, 'priority_breakdown': {}, 'immediate_items': 0, 
+                   'clone_items': 0, 'scheduled_items': 0}
+    
+    # PRIVATE METHODS
+    def _get_item_uuid(self, item) -> str:
+        """Safely extract UUID from item for logging"""
+        try:
+            if hasattr(item, 'item') and isinstance(item.item, dict):
+                return item.item.get('uuid', 'unknown')
+        except Exception:
+            pass
+        return 'unknown'
+    
+    def _emit_put_signals(self, item):
+        """Emit signals when item is added"""
+        try:
+            # Watch update signal
+            if hasattr(item, 'item') and isinstance(item.item, dict) and 'uuid' in item.item:
+                watch_check_update = signal('watch_check_update')
+                if watch_check_update:
+                    watch_check_update.send(watch_uuid=item.item['uuid'])
+            
+            # Queue length signal
+            if self.queue_length_signal:
+                self.queue_length_signal.send(length=self.qsize())
+                
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to emit put signals: {str(e)}")
+    
+    def _emit_get_signals(self):
+        """Emit signals when item is removed"""
+        try:
+            if self.queue_length_signal:
+                self.queue_length_signal.send(length=self.qsize())
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to emit get signals: {str(e)}")
+
+
+class NotificationQueue:
+    """
+    Ultra-reliable notification queue using pure janus.
+    
+    CRITICAL DESIGN NOTE: Both sync_q and async_q are required because:
+    - sync_q: Used by Flask routes, ticker threads, and other synchronous code
+    - async_q: Used by async workers and coroutines
+    
+    DO NOT REMOVE EITHER INTERFACE - they bridge different execution contexts.
+    See RecheckPriorityQueue docstring above for detailed explanation.
+    
+    Simple wrapper around janus with bulletproof error handling.
+    """
+    
+    def __init__(self, maxsize: int = 0):
+        try:
+            self._janus_queue = janus.Queue(maxsize=maxsize)
+            # BOTH interfaces required - see class docstring for why
+            self.sync_q = self._janus_queue.sync_q   # Flask routes, threads
+            self.async_q = self._janus_queue.async_q # Async workers
+            self.notification_event_signal = signal('notification_event')
+            logger.debug("NotificationQueue initialized successfully")
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to initialize NotificationQueue: {str(e)}")
+            raise
+    
+    def put(self, item: Dict[str, Any], block: bool = True, timeout: Optional[float] = None):
+        """Thread-safe sync put with signal emission"""
+        try:
+            self.sync_q.put(item, block=block, timeout=timeout)
+            self._emit_notification_signal(item)
+            logger.debug(f"Successfully queued notification: {item.get('uuid', 'unknown')}")
+            return True
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to put notification {item.get('uuid', 'unknown')}: {str(e)}")
+            return False
+    
+    async def async_put(self, item: Dict[str, Any]):
+        """Pure async put with signal emission"""
+        try:
+            await self.async_q.put(item)
+            self._emit_notification_signal(item)
+            logger.debug(f"Successfully async queued notification: {item.get('uuid', 'unknown')}")
+            return True
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to async put notification {item.get('uuid', 'unknown')}: {str(e)}")
+            return False
+    
+    def get(self, block: bool = True, timeout: Optional[float] = None):
+        """Thread-safe sync get"""
+        try:
+            return self.sync_q.get(block=block, timeout=timeout)
+        except queue.Empty as e:
+            raise e
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to get notification: {str(e)}")
+            raise e
+    
+    async def async_get(self):
+        """Pure async get"""
+        try:
+            return await self.async_q.get()
+        except queue.Empty as e:
+            raise e
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to async get notification: {str(e)}")
+            raise e
+    
+    def qsize(self) -> int:
+        """Get current queue size"""
+        try:
+            return self.sync_q.qsize()
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to get notification queue size: {str(e)}")
+            return 0
+    
+    def empty(self) -> bool:
+        """Check if queue is empty"""
+        return self.qsize() == 0
+    
+    def close(self):
+        """Close the janus queue"""
+        try:
+            self._janus_queue.close()
+            logger.debug("NotificationQueue closed successfully")
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to close NotificationQueue: {str(e)}")
+    
+    def _emit_notification_signal(self, item: Dict[str, Any]):
+        """Emit notification signal"""
+        try:
+            if self.notification_event_signal and isinstance(item, dict):
+                watch_uuid = item.get('uuid')
+                if watch_uuid:
+                    self.notification_event_signal.send(watch_uuid=watch_uuid)
+                else:
+                    self.notification_event_signal.send()
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to emit notification signal: {str(e)}")

changedetectionio/static/js/realtime.js

@@ -153,6 +153,7 @@ $(document).ready(function () {
 
                 // Tabs at bottom of list
                 $('#post-list-mark-views').toggleClass("has-unviewed", general_stats.has_unviewed);
+                $('#post-list-unread').toggleClass("has-unviewed", general_stats.has_unviewed);
                 $('#post-list-with-errors').toggleClass("has-error", general_stats.count_errors !== 0)
                 $('#post-list-with-errors a').text(`With errors (${ general_stats.count_errors })`);
 

changedetectionio/static/js/watch-settings.js

@@ -51,6 +51,7 @@ $(document).ready(function () {
         $('#notification_body').val('');
         $('#notification_format').val('System default');
         $('#notification_urls').val('');
+        $('#notification_muted_none').prop('checked', true); // in the case of a ternary field
         e.preventDefault();
     });
     $("#notification-token-toggle").click(function (e) {

changedetectionio/static/styles/scss/parts/_socket.scss

@@ -24,6 +24,9 @@ body.checking-now {
   #post-list-mark-views.has-unviewed {
     display: inline-block !important;
   }
+  #post-list-unread.has-unviewed {
+    display: inline-block !important;
+  }
 }
 
 

changedetectionio/static/styles/scss/parts/_widgets.scss

@@ -0,0 +1,115 @@
+
+// Ternary radio button group component
+.ternary-radio-group {
+  display: flex;
+  gap: 0;
+  border: 1px solid var(--color-grey-750);
+  border-radius: 4px;
+  overflow: hidden;
+  width: fit-content;
+  background: var(--color-background);
+
+  .ternary-radio-option {
+    position: relative;
+    cursor: pointer;
+    margin: 0;
+    display: flex;
+    align-items: center;
+
+    input[type="radio"] {
+      position: absolute;
+      opacity: 0;
+      width: 0;
+      height: 0;
+    }
+
+    .ternary-radio-label {
+      padding: 8px 16px;
+      background: var(--color-grey-900);
+      border: none;
+      border-right: 1px solid var(--color-grey-750);
+      font-size: 13px;
+      font-weight: 500;
+      color: var(--color-text);
+      transition: all 0.2s ease;
+      cursor: pointer;
+      display: block;
+      min-width: 60px;
+      text-align: center;
+    }
+
+    &:last-child .ternary-radio-label {
+      border-right: none;
+    }
+
+    input:checked + .ternary-radio-label {
+      background: var(--color-link);
+      color: var(--color-text-button);
+      font-weight: 600;
+
+      &.ternary-default {
+        background: var(--color-grey-600);
+        color: var(--color-text-button);
+      }
+
+      &:hover {
+        background: #1a7bc4;
+
+        &.ternary-default {
+          background: var(--color-grey-500);
+        }
+      }
+    }
+
+    &:hover .ternary-radio-label {
+      background: var(--color-grey-800);
+    }
+  }
+
+  @media (max-width: 480px) {
+    width: 100%;
+
+    .ternary-radio-label {
+      flex: 1;
+      min-width: auto;
+    }
+  }
+}
+
+// Standard radio button styling
+input[type="radio"].pure-radio:checked + label,
+input[type="radio"].pure-radio:checked {
+  background: var(--color-link);
+  color: var(--color-text-button);
+}
+
+html[data-darkmode="true"] {
+  .ternary-radio-group {
+    .ternary-radio-option {
+      .ternary-radio-label {
+        background: var(--color-grey-350);
+      }
+
+      &:hover .ternary-radio-label {
+        background: var(--color-grey-400);
+      }
+
+      input:checked + .ternary-radio-label {
+        background: var(--color-link);
+        color: var(--color-text-button);
+
+        &.ternary-default {
+          background: var(--color-grey-600);
+        }
+
+        &:hover {
+          background: #1a7bc4;
+
+          &.ternary-default {
+            background: var(--color-grey-500);
+          }
+        }
+      }
+    }
+  }
+}

changedetectionio/static/styles/scss/styles.scss

@@ -20,7 +20,7 @@
 @use "parts/lister_extra";
 @use "parts/socket";
 @use "parts/visualselector";
-
+@use "parts/widgets";
 
 body {
   color: var(--color-text);
@@ -1130,11 +1130,12 @@ ul {
 }
 
 #realtime-conn-error {
-  position: absolute;
+  position: fixed;
   bottom: 0;
-  left: 30px;
+  left: 0;
   background: var(--color-warning);
   padding: 10px;
   font-size: 0.8rem;
   color: #fff;
+  opacity: 0.8;
 }

changedetectionio/store.py

@@ -262,11 +262,6 @@ class ChangeDetectionStore:
         extras = deepcopy(self.data['watching'][uuid])
         new_uuid = self.add_watch(url=url, extras=extras)
         watch = self.data['watching'][new_uuid]
-
-        if self.data['settings']['application'].get('extract_title_as_title') or watch['extract_title_as_title']:
-            # Because it will be recalculated on the next fetch
-            self.data['watching'][new_uuid]['title'] = None
-
         return new_uuid
 
     def url_exists(self, url):
@@ -308,7 +303,6 @@ class ChangeDetectionStore:
                     'browser_steps',
                     'css_filter',
                     'extract_text',
-                    'extract_title_as_title',
                     'headers',
                     'ignore_text',
                     'include_filters',
@@ -323,6 +317,7 @@ class ChangeDetectionStore:
                     'title',
                     'trigger_text',
                     'url',
+                    'use_page_title_in_list',
                     'webdriver_js_execute_code',
                 ]:
                     if res.get(k):
@@ -973,6 +968,16 @@ class ChangeDetectionStore:
                         f_d.write(zlib.compress(f_j.read()))
                         os.unlink(json_path)
 
+    def update_20(self):
+        for uuid, watch in self.data['watching'].items():
+            if self.data['watching'][uuid].get('extract_title_as_title'):
+                self.data['watching'][uuid]['use_page_title_in_list'] = self.data['watching'][uuid].get('extract_title_as_title')
+                del self.data['watching'][uuid]['extract_title_as_title']
+
+        if self.data['settings']['application'].get('extract_title_as_title'):
+            self.data['settings']['application']['ui']['use_page_title_in_list'] = self.data['settings']['application'].get('extract_title_as_title')
+
+
     def add_notification_url(self, notification_url):
         
         logger.debug(f">>> Adding new notification_url - '{notification_url}'")

changedetectionio/templates/_common_fields.html

@@ -70,7 +70,7 @@
                                     </tr>
                                     <tr>
                                         <td><code>{{ '{{watch_title}}' }}</code></td>
-                                        <td>The title of the watch.</td>
+                                        <td>The page title of the watch, uses &lt;title&gt; if not set, falls back to URL</td>
                                     </tr>
                                     <tr>
                                         <td><code>{{ '{{watch_tag}}' }}</code></td>

changedetectionio/templates/_helpers.html

@@ -1,6 +1,13 @@
 {% macro render_field(field) %}
-  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
+  <div {% if field.errors or field.top_errors %} class="error" {% endif %}>{{ field.label }}</div>
-  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  <div {% if field.errors or field.top_errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.top_errors %}
+    <ul class="errors top-errors">
+    {% for error in field.top_errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
   {% if field.errors %}
     <ul class=errors>
     {% for error in field.errors %}
@@ -24,6 +31,23 @@
   </div>
 {% endmacro %}
 
+{% macro render_ternary_field(field, BooleanField=false) %}
+  {% if BooleanField %}
+    {% set _ = field.__setattr__('boolean_mode', true) %}
+  {% endif %}
+  <div class="ternary-field {% if field.errors %} error {% endif %}">
+    <div class="ternary-field-label">{{ field.label }}</div>
+    <div class="ternary-field-widget">{{ field(**kwargs)|safe }}</div>
+    {% if field.errors %}
+      <ul class=errors>
+      {% for error in field.errors %}
+        <li>{{ error }}</li>
+      {% endfor %}
+      </ul>
+    {% endif %}
+  </div>
+{% endmacro %}
+
 
 {% macro render_simple_field(field) %}
   <span class="label {% if field.errors %}error{% endif %}">{{ field.label }}</span>

changedetectionio/templates/base.html

@@ -5,6 +5,7 @@
     <meta charset="utf-8" >
     <meta name="viewport" content="width=device-width, initial-scale=1.0" >
     <meta name="description" content="Self hosted website change detection." >
+    <meta name="robots" content="noindex">
     <title>Change Detection{{extra_title}}</title>
     {% if app_rss_token %}
       <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag_uuid %}- {{active_tag.title}}{% endif %}" href="{{ url_for('rss.feed', tag=active_tag_uuid , token=app_rss_token)}}" >
@@ -40,7 +41,7 @@
     {% endif %}
   </head>
 
-  <body class="">
+  <body class="{{extra_classes}}">
     <div class="header">
     <div class="pure-menu-fixed" style="width: 100%;">
       <div class="home-menu pure-menu pure-menu-horizontal" id="nav-menu">
@@ -236,7 +237,7 @@
     <script src="{{url_for('static_content', group='js', filename='toggle-theme.js')}}" defer></script>
 
     <div id="checking-now-fixed-tab" style="display: none;"><span class="spinner"></span><span>&nbsp;Checking now</span></div>
-    <div id="realtime-conn-error" style="display:none">Offline</div>
+    <div id="realtime-conn-error" style="display:none">Real-time updates offline</div>
   </body>
 
 </html>

changedetectionio/tests/custom_browser_url/test_custom_browser_url.py

@@ -55,7 +55,8 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
                   "tags": "",
                   "headers": "",
                   'fetch_backend': f"extra_browser_{custom_browser_name}",
-                  'webdriver_js_execute_code': ''
+                  'webdriver_js_execute_code': '',
+                  "time_between_check_use_default": "y"
             },
             follow_redirects=True
         )

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -28,6 +28,7 @@ def test_execute_custom_js(client, live_server, measure_memory_usage):
             'fetch_backend': "html_webdriver",
             'webdriver_js_execute_code': 'document.querySelector("button[name=test-button]").click();',
             'headers': "testheader: yes\buser-agent: MyCustomAgent",
+            "time_between_check_use_default": "y",
         },
         follow_redirects=True
     )

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -27,6 +27,7 @@ def test_preferred_proxy(client, live_server, measure_memory_usage):
                 "proxy": "proxy-two",
                 "tags": "",
                 "url": url,
+                "time_between_check_use_default": "y",
               },
         follow_redirects=True
     )

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -62,6 +62,7 @@ def test_noproxy_option(client, live_server, measure_memory_usage):
                 "proxy": "no-proxy",
                 "tags": "",
                 "url": url,
+                "time_between_check_use_default": "y",
               },
         follow_redirects=True
     )

changedetectionio/tests/proxy_list/test_proxy_noconnect.py

@@ -44,6 +44,7 @@ def test_proxy_noconnect_custom(client, live_server, measure_memory_usage):
         "url": test_url,
         "fetch_backend": "html_webdriver" if os.getenv('PLAYWRIGHT_DRIVER_URL') or os.getenv("WEBDRIVER_URL") else "html_requests",
         "proxy": "ui-0custom-test-proxy",
+        "time_between_check_use_default": "y",
     }
 
     res = client.post(

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -66,6 +66,7 @@ def test_socks5(client, live_server, measure_memory_usage):
             "proxy": "ui-0socks5proxy",
             "tags": "",
             "url": test_url,
+            "time_between_check_use_default": "y",
         },
         follow_redirects=True
     )

changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py

@@ -53,6 +53,7 @@ def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage)
             "proxy": "socks5proxy",
             "tags": "",
             "url": test_url,
+            "time_between_check_use_default": "y",
         },
         follow_redirects=True
     )

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -157,7 +157,8 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
         data={
             "url": test_url,
             "notification_format": 'HTML',
-            'fetch_backend': "html_requests"},
+            'fetch_backend': "html_requests",
+            "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -61,7 +61,8 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
         data={"trigger_text": 'The golden line',
               "url": test_url,
               'fetch_backend': "html_requests",
-              'filter_text_removed': 'y'},
+              'filter_text_removed': 'y',
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -154,7 +155,8 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
               'processor': 'text_json_diff',
               'fetch_backend': "html_requests",
               'filter_text_removed': '',
-              'filter_text_added': 'y'},
+              'filter_text_added': 'y',
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_api.py

@@ -311,7 +311,7 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
                                  "value": "."  # contains anything
                              }
                          ],
-                         "conditions_match_logic": "ALL"
+                         "conditions_match_logic": "ALL",
                          }
                         ),
         headers={'content-type': 'application/json', 'x-api-key': api_key},
@@ -328,6 +328,7 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
     )
 
     watch_uuid = list(res.json.keys())[0]
+    assert not res.json[watch_uuid].get('viewed'), 'A newly created watch can only be unviewed'
 
     # Check in the edit page just to be sure
     res = client.get(
@@ -341,7 +342,12 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
     res = client.put(
         url_for("watch", uuid=watch_uuid),
         headers={'x-api-key': api_key, 'content-type': 'application/json'},
-        data=json.dumps({"title": "new title", 'time_between_check': {'minutes': 552}, 'headers': {'cookie': 'all eaten'}}),
+        data=json.dumps({
+            "title": "new title",
+            'time_between_check': {'minutes': 552},
+            'headers': {'cookie': 'all eaten'},
+            'last_viewed': int(time.time())
+        }),
     )
     assert res.status_code == 200, "HTTP PUT update was sent OK"
 
@@ -351,6 +357,7 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
         headers={'x-api-key': api_key}
     )
     assert res.json.get('title') == 'new title'
+    assert res.json.get('viewed'), 'With the timestamp greater than "changed" a watch can be updated to viewed'
 
     # Check in the edit page just to be sure
     res = client.get(
@@ -383,13 +390,13 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
 
 
 def test_api_import(client, live_server, measure_memory_usage):
-    
+
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
     res = client.post(
         url_for("import") + "?tag=import-test",
         data='https://website1.com\r\nhttps://website2.com',
-        headers={'x-api-key': api_key},
+        headers={'x-api-key': api_key, 'content-type': 'text/plain'},
         follow_redirects=True
     )
 

changedetectionio/tests/test_api_openapi.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+"""
+OpenAPI validation tests for ChangeDetection.io API
+
+This test file specifically verifies that OpenAPI validation is working correctly
+by testing various scenarios that should trigger validation errors.
+"""
+
+import time
+import json
+from flask import url_for
+from .util import live_server_setup, wait_for_all_checks
+
+
+def test_openapi_validation_invalid_content_type_on_create_watch(client, live_server, measure_memory_usage):
+    """Test that creating a watch with invalid content-type triggers OpenAPI validation error."""
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+
+    # Try to create a watch with JSON data but without proper content-type header
+    res = client.post(
+        url_for("createwatch"),
+        data=json.dumps({"url": "https://example.com", "title": "Test Watch"}),
+        headers={'x-api-key': api_key},  # Missing 'content-type': 'application/json'
+        follow_redirects=True
+    )
+
+    # Should get 400 error due to OpenAPI validation failure
+    assert res.status_code == 400, f"Expected 400 but got {res.status_code}"
+    assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
+
+
+def test_openapi_validation_missing_required_field_create_watch(client, live_server, measure_memory_usage):
+    """Test that creating a watch without required URL field triggers OpenAPI validation error."""
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+
+    # Try to create a watch without the required 'url' field
+    res = client.post(
+        url_for("createwatch"),
+        data=json.dumps({"title": "Test Watch Without URL"}),  # Missing required 'url' field
+        headers={'x-api-key': api_key, 'content-type': 'application/json'},
+        follow_redirects=True
+    )
+
+    # Should get 400 error due to missing required field
+    assert res.status_code == 400, f"Expected 400 but got {res.status_code}"
+    assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
+
+
+def test_openapi_validation_invalid_field_in_request_body(client, live_server, measure_memory_usage):
+    """Test that including invalid fields triggers OpenAPI validation error."""
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+
+    # First create a valid watch
+    res = client.post(
+        url_for("createwatch"),
+        data=json.dumps({"url": "https://example.com", "title": "Test Watch"}),
+        headers={'x-api-key': api_key, 'content-type': 'application/json'},
+        follow_redirects=True
+    )
+    assert res.status_code == 201, "Watch creation should succeed"
+
+    # Get the watch list to find the UUID
+    res = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    watch_uuid = list(res.json.keys())[0]
+
+    # Now try to update the watch with an invalid field
+    res = client.put(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key, 'content-type': 'application/json'},
+        data=json.dumps({
+            "title": "Updated title",
+            "invalid_field_that_doesnt_exist": "this should cause validation error"
+        }),
+    )
+
+    # Should get 400 error due to invalid field (this will be caught by internal validation)
+    # Note: This tests the flow where OpenAPI validation passes but internal validation catches it
+    assert res.status_code == 400, f"Expected 400 but got {res.status_code}"
+    assert b"Additional properties are not allowed" in res.data, "Should contain validation error about additional properties"
+
+
+def test_openapi_validation_import_wrong_content_type(client, live_server, measure_memory_usage):
+    """Test that import endpoint with wrong content-type triggers OpenAPI validation error."""
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+
+    # Try to import URLs with JSON content-type instead of text/plain
+    res = client.post(
+        url_for("import") + "?tag=test-import",
+        data='https://website1.com\nhttps://website2.com',
+        headers={'x-api-key': api_key, 'content-type': 'application/json'},  # Wrong content-type
+        follow_redirects=True
+    )
+
+    # Should get 400 error due to content-type mismatch
+    assert res.status_code == 400, f"Expected 400 but got {res.status_code}"
+    assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
+
+
+def test_openapi_validation_import_correct_content_type_succeeds(client, live_server, measure_memory_usage):
+    """Test that import endpoint with correct content-type succeeds (positive test)."""
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+
+    # Import URLs with correct text/plain content-type
+    res = client.post(
+        url_for("import") + "?tag=test-import",
+        data='https://website1.com\nhttps://website2.com',
+        headers={'x-api-key': api_key, 'content-type': 'text/plain'},  # Correct content-type
+        follow_redirects=True
+    )
+
+    # Should succeed
+    assert res.status_code == 200, f"Expected 200 but got {res.status_code}"
+    assert len(res.json) == 2, "Should import 2 URLs"
+
+
+def test_openapi_validation_get_requests_bypass_validation(client, live_server, measure_memory_usage):
+    """Test that GET requests bypass OpenAPI validation entirely."""
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+
+    # Disable API token requirement first
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={
+            "requests-time_between_check-minutes": 180,
+            "application-fetch_backend": "html_requests",
+            "application-api_access_token_enabled": ""
+        },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Make GET request to list watches - should succeed even without API key or content-type
+    res = client.get(url_for("createwatch"))  # No headers needed for GET
+    assert res.status_code == 200, f"GET requests should succeed without OpenAPI validation, got {res.status_code}"
+
+    # Should return JSON with watch list (empty in this case)
+    assert isinstance(res.json, dict), "Should return JSON dictionary for watch list"
+
+
+def test_openapi_validation_create_tag_missing_required_title(client, live_server, measure_memory_usage):
+    """Test that creating a tag without required title triggers OpenAPI validation error."""
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+
+    # Try to create a tag without the required 'title' field
+    res = client.post(
+        url_for("tag"),
+        data=json.dumps({"notification_urls": ["mailto:test@example.com"]}),  # Missing required 'title' field
+        headers={'x-api-key': api_key, 'content-type': 'application/json'},
+        follow_redirects=True
+    )
+
+    # Should get 400 error due to missing required field
+    assert res.status_code == 400, f"Expected 400 but got {res.status_code}"
+    assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
+
+
+def test_openapi_validation_watch_update_allows_partial_updates(client, live_server, measure_memory_usage):
+    """Test that watch updates allow partial updates without requiring all fields (positive test)."""
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+
+    # First create a valid watch
+    res = client.post(
+        url_for("createwatch"),
+        data=json.dumps({"url": "https://example.com", "title": "Test Watch"}),
+        headers={'x-api-key': api_key, 'content-type': 'application/json'},
+        follow_redirects=True
+    )
+    assert res.status_code == 201, "Watch creation should succeed"
+
+    # Get the watch list to find the UUID
+    res = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    watch_uuid = list(res.json.keys())[0]
+
+    # Update only the title (partial update) - should succeed
+    res = client.put(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key, 'content-type': 'application/json'},
+        data=json.dumps({"title": "Updated Title Only"}),  # Only updating title, not URL
+    )
+
+    # Should succeed because UpdateWatch schema allows partial updates
+    assert res.status_code == 200, f"Partial updates should succeed, got {res.status_code}"
+
+    # Verify the update worked
+    res = client.get(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key}
+    )
+    assert res.status_code == 200
+    assert res.json.get('title') == 'Updated Title Only', "Title should be updated"
+    assert res.json.get('url') == 'https://example.com', "URL should remain unchanged"

changedetectionio/tests/test_api_tags.py

@@ -1,15 +1,18 @@
 #!/usr/bin/env python3
 
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from .util import live_server_setup, wait_for_all_checks, set_original_response
 import json
+import time
 
 def test_api_tags_listing(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
     tag_title = 'Test Tag'
 
-    # Get a listing
+
+    set_original_response()
+
     res = client.get(
         url_for("tags"),
         headers={'x-api-key': api_key}
@@ -104,6 +107,8 @@ def test_api_tags_listing(client, live_server, measure_memory_usage):
     assert res.status_code == 201
     watch_uuid = res.json.get('uuid')
 
+
+    wait_for_all_checks()
     # Verify tag is associated with watch by name if need be
     res = client.get(
         url_for("watch", uuid=watch_uuid),
@@ -112,6 +117,21 @@ def test_api_tags_listing(client, live_server, measure_memory_usage):
     assert res.status_code == 200
     assert new_tag_uuid in res.json.get('tags', [])
 
+    # Check recheck by tag
+    before_check_time = live_server.app.config['DATASTORE'].data['watching'][watch_uuid].get('last_checked')
+    time.sleep(1)
+    res = client.get(
+       url_for("tag", uuid=new_tag_uuid) + "?recheck=true",
+       headers={'x-api-key': api_key}
+    )
+    wait_for_all_checks()
+    assert res.status_code == 200
+    assert b'OK, 1 watches' in res.data
+
+    after_check_time = live_server.app.config['DATASTORE'].data['watching'][watch_uuid].get('last_checked')
+
+    assert before_check_time != after_check_time
+
     # Delete tag
     res = client.delete(
         url_for("tag", uuid=new_tag_uuid),
@@ -141,3 +161,6 @@ def test_api_tags_listing(client, live_server, measure_memory_usage):
         headers={'x-api-key': api_key},
     )
     assert res.status_code == 204
+
+
+

changedetectionio/tests/test_auth.py

@@ -23,7 +23,7 @@ def test_basic_auth(client, live_server, measure_memory_usage):
     # Check form validation
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"include_filters": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_backend.py

@@ -89,7 +89,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     assert b'CDATA' in res.data
 
     assert expected_url.encode('utf-8') in res.data
-
+#
     # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times
     res = client.get(url_for("ui.ui_views.diff_history_page", uuid=uuid))
     assert b'selected=""' in res.data, "Confirm diff history page loaded"
@@ -104,26 +104,34 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     wait_for_all_checks(client)
 
-    # Do this a few times.. ensures we dont accidently set the status
-    for n in range(2):
-        client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
+    # Do this a few times.. ensures we don't accidently set the status
+    for n in range(2):
+        res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
         # Give the thread time to pick it up
         wait_for_all_checks(client)
 
         # It should report nothing found (no new 'unviewed' class)
         res = client.get(url_for("watchlist.index"))
+
+
         assert b'unviewed' not in res.data
         assert b'class="has-unviewed' not in res.data
-        assert b'head title' not in res.data  # Should not be present because this is off by default
+        assert b'head title' in res.data  # Should be ON by default
         assert b'test-endpoint' in res.data
 
-    set_original_response()
+    # Recheck it but only with a title change, content wasnt changed
+    set_original_response(extra_title=" and more")
 
-    # Enable auto pickup of <title> in settings
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+    res = client.get(url_for("watchlist.index"))
+    assert b'head title and more' in res.data
+
+    # disable <title> pickup
     res = client.post(
         url_for("settings.settings_page"),
-        data={"application-extract_title_as_title": "1", "requests-time_between_check-minutes": 180,
+        data={"application-ui-use_page_title_in_list": "", "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
     )
@@ -134,16 +142,14 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     res = client.get(url_for("watchlist.index"))
     assert b'unviewed' in res.data
     assert b'class="has-unviewed' in res.data
+    assert b'head title' not in res.data  # should now be off
 
-    # It should have picked up the <title>
-    assert b'head title' in res.data
 
     # Be sure the last_viewed is going to be greater than the last snapshot
     time.sleep(1)
 
     # hit the mark all viewed link
     res = client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
-    time.sleep(0.2)
 
     assert b'class="has-unviewed' not in res.data
     assert b'unviewed' not in res.data

changedetectionio/tests/test_block_while_text_present.py

@@ -86,7 +86,8 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"text_should_not_be_present": ignore_text,
               "url": test_url,
-              'fetch_backend': "html_requests"
+              'fetch_backend': "html_requests",
+              "time_between_check_use_default": "y"
               },
         follow_redirects=True
     )

changedetectionio/tests/test_conditions.py

@@ -105,6 +105,7 @@ def test_conditions_with_text_and_number(client, live_server):
             "conditions-5-operator": "contains_regex",
             "conditions-5-field": "page_filtered_text",
             "conditions-5-value": "\d",
+            "time_between_check_use_default": "y",
         },
         follow_redirects=True
     )
@@ -288,7 +289,8 @@ def test_lev_conditions_plugin(client, live_server, measure_memory_usage):
             "conditions_match_logic": CONDITIONS_MATCH_LOGIC_DEFAULT,  # ALL = AND logic
             "conditions-0-field": "levenshtein_ratio",
             "conditions-0-operator": "<",
-            "conditions-0-value": "0.8" # needs to be more of a diff to trigger a change
+            "conditions-0-value": "0.8", # needs to be more of a diff to trigger a change
+            "time_between_check_use_default": "y"
         },
         follow_redirects=True
     )

changedetectionio/tests/test_css_selector.py

@@ -95,7 +95,7 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
     # Add our URL to the import page
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -154,7 +154,8 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage):
               "url": test_url,
               "tags": "",
               "headers": "",
-              'fetch_backend': "html_requests"},
+              'fetch_backend': "html_requests",
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -208,7 +209,8 @@ def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usa
               "url": test_url,
               "tags": "",
               "headers": "",
-              'fetch_backend': "html_requests"},
+              'fetch_backend': "html_requests",
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_element_removal.py

@@ -171,6 +171,7 @@ def test_element_removal_full(client, live_server, measure_memory_usage):
             "tags": "",
             "headers": "",
             "fetch_backend": "html_requests",
+            "time_between_check_use_default": "y",
         },
         follow_redirects=True,
     )
@@ -245,6 +246,7 @@ body > table > tr:nth-child(3) > td:nth-child(3)""",
                 "url": test_url,
                 "tags": "",
                 "fetch_backend": "html_requests",
+                "time_between_check_use_default": "y",
             },
             follow_redirects=True,
         )

changedetectionio/tests/test_errorhandling.py

@@ -127,7 +127,8 @@ def test_low_level_errors_clear_correctly(client, live_server, measure_memory_us
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={
             "url": test_url,
-            "fetch_backend": "html_requests"},
+            "fetch_backend": "html_requests",
+            "time_between_check_use_default": "y"},
         follow_redirects=True
     )
 

changedetectionio/tests/test_extract_regex.py

@@ -95,7 +95,8 @@ def test_check_filter_multiline(client, live_server, measure_memory_usage):
               "url": test_url,
               "tags": "",
               "headers": "",
-              'fetch_backend': "html_requests"
+              'fetch_backend': "html_requests",
+              "time_between_check_use_default": "y"
               },
         follow_redirects=True
     )
@@ -149,7 +150,8 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag
               "url": test_url,
               "tags": "",
               "headers": "",
-              'fetch_backend': "html_requests"
+              'fetch_backend': "html_requests",
+              "time_between_check_use_default": "y"
               },
         follow_redirects=True
     )
@@ -222,7 +224,8 @@ def test_regex_error_handling(client, live_server, measure_memory_usage):
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"extract_text": '/something bad\d{3/XYZ',
               "url": test_url,
-              "fetch_backend": "html_requests"},
+              "fetch_backend": "html_requests",
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
 

changedetectionio/tests/test_filter_exist_changes.py

@@ -94,7 +94,8 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
         "title": "my title",
         "headers": "",
         "include_filters": '.ticket-available',
-        "fetch_backend": "html_requests"})
+        "fetch_backend": "html_requests",
+        "time_between_check_use_default": "y"})
 
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),

changedetectionio/tests/test_filter_failure_notification.py

@@ -72,6 +72,7 @@ def run_filter_test(client, live_server, content_filter):
                   "notification_format": "Text",
                   "fetch_backend": "html_requests",
                   "filter_failure_notification_send": 'y',
+                  "time_between_check_use_default": "y",
                   "headers": "",
                   "tags": "my tag",
                   "title": "my title 123",

changedetectionio/tests/test_group.py

@@ -424,7 +424,8 @@ def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measu
             "url": test_url,
             "tags": "test-tag-keep-order",
             "headers": "",
-            'fetch_backend': "html_requests"},
+            'fetch_backend': "html_requests",
+            "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_ignore_text.py

@@ -111,7 +111,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     # Add our URL to the import page
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"ignore_text": ignore_text, "url": test_url, 'fetch_backend': "html_requests"},
+        data={"ignore_text": ignore_text, "url": test_url, 'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -205,7 +205,7 @@ def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
     #Adding some ignore text should not trigger a change
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"ignore_text": "something irrelevent but just to check", "url": test_url, 'fetch_backend': "html_requests"},
+        data={"ignore_text": "something irrelevent but just to check", "url": test_url, 'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_ignorestatuscode.py

@@ -108,7 +108,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server, measu
     # Add our URL to the import page
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"ignore_status_codes": "y", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"ignore_status_codes": "y", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -257,7 +257,8 @@ def check_json_filter(json_filter, client, live_server):
               "url": test_url,
               "tags": "",
               "headers": "",
-              "fetch_backend": "html_requests"
+              "fetch_backend": "html_requests",
+              "time_between_check_use_default": "y"
               },
         follow_redirects=True
     )
@@ -328,7 +329,8 @@ def check_json_filter_bool_val(json_filter, client, live_server):
               "url": test_url,
               "tags": "",
               "headers": "",
-              "fetch_backend": "html_requests"
+              "fetch_backend": "html_requests",
+              "time_between_check_use_default": "y"
               },
         follow_redirects=True
     )
@@ -393,7 +395,8 @@ def check_json_ext_filter(json_filter, client, live_server):
               "url": test_url,
               "tags": "",
               "headers": "",
-              "fetch_backend": "html_requests"
+              "fetch_backend": "html_requests",
+              "time_between_check_use_default": "y"
               },
         follow_redirects=True
     )

changedetectionio/tests/test_live_preview.py

@@ -38,6 +38,7 @@ def test_content_filter_live_preview(client, live_server, measure_memory_usage):
             "ignore_text": "something to ignore",
             "trigger_text": "something to trigger",
             "url": test_url,
+            "time_between_check_use_default": "y",
         },
         follow_redirects=True
     )

changedetectionio/tests/test_notification.py

@@ -108,7 +108,8 @@ def test_check_notification(client, live_server, measure_memory_usage):
         "tags": "my tag, my second tag",
         "title": "my title",
         "headers": "",
-        "fetch_backend": "html_requests"})
+        "fetch_backend": "html_requests",
+        "time_between_check_use_default": "y"})
 
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
@@ -225,7 +226,8 @@ def test_check_notification(client, live_server, measure_memory_usage):
         "notification_title": '',
         "notification_body": '',
         "notification_format": default_notification_format,
-        "fetch_backend": "html_requests"},
+        "fetch_backend": "html_requests",
+        "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_notification_errors.py

@@ -36,7 +36,8 @@ def test_check_notification_error_handling(client, live_server, measure_memory_u
               "title": "",
               "headers": "",
               "time_between_check-minutes": "180",
-              "fetch_backend": "html_requests"},
+              "fetch_backend": "html_requests",
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_request.py

@@ -44,7 +44,8 @@ def test_headers_in_request(client, live_server, measure_memory_usage):
               "url": test_url,
               "tags": "",
               "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
-              "headers": "jinja2:{{ 1+1 }}\nxxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
+              "headers": "jinja2:{{ 1+1 }}\nxxx:ooo\ncool:yeah\r\ncookie:"+cookie_header,
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -109,7 +110,8 @@ def test_body_in_request(client, live_server, measure_memory_usage):
               "tags": "",
               "method": "POST",
               "fetch_backend": "html_requests",
-              "body": "something something"},
+              "body": "something something",
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -126,7 +128,8 @@ def test_body_in_request(client, live_server, measure_memory_usage):
               "tags": "",
               "method": "POST",
               "fetch_backend": "html_requests",
-              "body": body_value},
+              "body": body_value,
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -172,7 +175,8 @@ def test_body_in_request(client, live_server, measure_memory_usage):
               "tags": "",
               "method": "GET",
               "fetch_backend": "html_requests",
-              "body": "invalid"},
+              "body": "invalid",
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Body must be empty when Request Method is set to GET" in res.data
@@ -211,7 +215,8 @@ def test_method_in_request(client, live_server, measure_memory_usage):
             "url": test_url,
             "tags": "",
             "fetch_backend": "html_requests",
-            "method": "invalid"},
+            "method": "invalid",
+            "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Not a valid choice" in res.data
@@ -223,7 +228,8 @@ def test_method_in_request(client, live_server, measure_memory_usage):
             "url": test_url,
             "tags": "",
             "fetch_backend": "html_requests",
-            "method": "PATCH"},
+            "method": "PATCH",
+            "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -297,7 +303,8 @@ def test_ua_global_override(client, live_server, measure_memory_usage):
             "tags": "testtag",
             "fetch_backend": 'html_requests',
             # Important - also test case-insensitive
-            "headers": "User-AGent: agent-from-watch"},
+            "headers": "User-AGent: agent-from-watch",
+            "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -365,7 +372,8 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
             "url": test_url,
             "tags": "testtag",
             "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
-            "headers": "xxx:ooo\ncool:yeah\r\n"},
+            "headers": "xxx:ooo\ncool:yeah\r\n",
+            "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -440,7 +448,8 @@ def test_headers_validation(client, live_server):
         data={
             "url": test_url,
             "fetch_backend": 'html_requests',
-            "headers": "User-AGent agent-from-watch\r\nsadfsadfsadfsdaf\r\n:foobar"},
+            "headers": "User-AGent agent-from-watch\r\nsadfsadfsadfsdaf\r\n:foobar",
+            "time_between_check_use_default": "y"},
         follow_redirects=True
     )
 

changedetectionio/tests/test_restock_itemprop.py

@@ -121,7 +121,7 @@ def test_itemprop_price_change(client, live_server):
     set_original_response(props_markup=instock_props[0], price='120.45')
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"restock_settings-follow_price_changes": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"restock_settings-follow_price_changes": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -155,7 +155,8 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
         "url": test_url,
         "headers": "",
         "time_between_check-hours": 5,
-        'fetch_backend': "html_requests"
+        'fetch_backend': "html_requests",
+        "time_between_check_use_default": "y"
     }
     data.update(extra_watch_edit_form)
     res = client.post(
@@ -278,7 +279,8 @@ def test_itemprop_percent_threshold(client, live_server):
               "url": test_url,
               "tags": "",
               "headers": "",
-              'fetch_backend': "html_requests"
+              'fetch_backend': "html_requests",
+              "time_between_check_use_default": "y"
               },
         follow_redirects=True
     )

changedetectionio/tests/test_rss.py

@@ -158,6 +158,7 @@ def test_rss_xpath_filtering(client, live_server, measure_memory_usage):
                 "proxy": "no-proxy",
                 "tags": "",
                 "url": test_url,
+                "time_between_check_use_default": "y",
               },
         follow_redirects=True
     )

changedetectionio/tests/test_scheduler.py

@@ -1,10 +1,13 @@
 #!/usr/bin/env python3
 
 import time
+from copy import copy
 from datetime import datetime, timezone
 from zoneinfo import ZoneInfo
 from flask import url_for
 from .util import  live_server_setup, wait_for_all_checks, extract_UUID_from_client
+from ..forms import REQUIRE_ATLEAST_ONE_TIME_PART_MESSAGE_DEFAULT, REQUIRE_ATLEAST_ONE_TIME_PART_WHEN_NOT_GLOBAL_DEFAULT
+
 
 # def test_setup(client, live_server):
    #  live_server_setup(live_server) # Setup on conftest per function
@@ -42,11 +45,12 @@ def test_check_basic_scheduler_functionality(client, live_server, measure_memory
     uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
-
+    last_check = copy(live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'])
     tpl = {
         "time_schedule_limit-XXX-start_time": "00:00",
         "time_schedule_limit-XXX-duration-hours": 24,
         "time_schedule_limit-XXX-duration-minutes": 0,
+        "time_between_check-seconds": 1,
         "time_schedule_limit-XXX-enabled": '',  # All days are turned off
         "time_schedule_limit-enabled": 'y',  # Scheduler is enabled, all days however are off.
     }
@@ -58,13 +62,13 @@ def test_check_basic_scheduler_functionality(client, live_server, measure_memory
             new_key = key.replace("XXX", day)
             scheduler_data[new_key] = value
 
-    last_check = live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked']
     data = {
         "url": test_url,
-        "fetch_backend": "html_requests"
+        "fetch_backend": "html_requests",
+        "time_between_check_use_default": "" # no
     }
     data.update(scheduler_data)
-
+    time.sleep(1)
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data=data,
@@ -77,6 +81,7 @@ def test_check_basic_scheduler_functionality(client, live_server, measure_memory
 
     # "Edit" should not trigger a check because it's not enabled in the schedule.
     time.sleep(2)
+    # "time_schedule_limit-XXX-enabled": '',  # All days are turned off, therefor, nothing should happen here..
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] == last_check
 
     # Enabling today in Kiritimati should work flawless
@@ -177,3 +182,44 @@ def test_check_basic_global_scheduler_functionality(client, live_server, measure
     # Cleanup everything
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
+
+
+def test_validation_time_interval_field(client, live_server, measure_memory_usage):
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("imports.import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+
+    res = client.post(
+        url_for("ui.ui_edit.edit_page", uuid="first"),
+        data={"trigger_text": 'The golden line',
+              "url": test_url,
+              'fetch_backend': "html_requests",
+              'filter_text_removed': 'y',
+              "time_between_check_use_default": ""
+              },
+        follow_redirects=True
+    )
+
+    assert REQUIRE_ATLEAST_ONE_TIME_PART_WHEN_NOT_GLOBAL_DEFAULT.encode('utf-8') in res.data
+
+    # Now set atleast something
+
+    res = client.post(
+        url_for("ui.ui_edit.edit_page", uuid="first"),
+        data={"trigger_text": 'The golden line',
+              "url": test_url,
+              'fetch_backend': "html_requests",
+              "time_between_check-minutes": 1,
+              "time_between_check_use_default": ""
+              },
+        follow_redirects=True
+    )
+
+    assert REQUIRE_ATLEAST_ONE_TIME_PART_WHEN_NOT_GLOBAL_DEFAULT.encode('utf-8') not in res.data
+
+

changedetectionio/tests/test_search.py

@@ -27,7 +27,7 @@ def test_basic_search(client, live_server, measure_memory_usage):
 
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"title": "xxx-title", "url": urls[0], "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"title": "xxx-title", "url": urls[0], "tags": "", "headers": "", 'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -62,7 +62,7 @@ def test_search_in_tag_limit(client, live_server, measure_memory_usage):
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"title": "xxx-title", "url": urls[0].split(' ')[0], "tags": urls[0].split(' ')[1], "headers": "",
-              'fetch_backend': "html_requests"},
+              'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_security.py

@@ -41,7 +41,8 @@ def test_bad_access(client, live_server, measure_memory_usage):
               "tags": "",
               "method": "GET",
               "fetch_backend": "html_requests",
-              "body": ""},
+              "body": "",
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
 
@@ -150,7 +151,8 @@ def test_xss_watch_last_error(client, live_server, measure_memory_usage):
         data={
             "include_filters": '<a href="https://foobar"></a><script>alert(123);</script>',
             "url": url_for('test_endpoint', _external=True),
-            'fetch_backend': "html_requests"
+            'fetch_backend': "html_requests",
+            "time_between_check_use_default": "y"
         },
         follow_redirects=True
     )

changedetectionio/tests/test_share_watch.py

@@ -29,7 +29,7 @@ def test_share_watch(client, live_server, measure_memory_usage):
     # Add our URL to the import page
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_source.py

@@ -77,7 +77,7 @@ def test_check_ignore_elements(client, live_server, measure_memory_usage):
 
     client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"include_filters": 'span,p', "url": test_url, "tags": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
+        data={"include_filters": 'span,p', "url": test_url, "tags": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
 

changedetectionio/tests/test_trigger.py

@@ -81,7 +81,8 @@ def test_trigger_functionality(client, live_server, measure_memory_usage):
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"trigger_text": trigger_text,
               "url": test_url,
-              "fetch_backend": "html_requests"},
+              "fetch_backend": "html_requests",
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_trigger_regex.py

@@ -49,7 +49,8 @@ def test_trigger_regex_functionality(client, live_server, measure_memory_usage):
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"trigger_text": '/something \d{3}/',
               "url": test_url,
-              "fetch_backend": "html_requests"},
+              "fetch_backend": "html_requests",
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     wait_for_all_checks(client)

changedetectionio/tests/test_trigger_regex_with_filter.py

@@ -50,7 +50,8 @@ def test_trigger_regex_functionality_with_filter(client, live_server, measure_me
         data={"trigger_text": "/cool.stuff/",
               "url": test_url,
               "include_filters": '#in-here',
-              "fetch_backend": "html_requests"},
+              "fetch_backend": "html_requests",
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
 

changedetectionio/tests/test_unique_lines.py

@@ -92,7 +92,8 @@ def test_unique_lines_functionality(client, live_server, measure_memory_usage):
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"check_unique_lines": "y",
               "url": test_url,
-              "fetch_backend": "html_requests"},
+              "fetch_backend": "html_requests",
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -140,7 +141,8 @@ def test_sort_lines_functionality(client, live_server, measure_memory_usage):
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"sort_text_alphabetically": "n",
               "url": test_url,
-              "fetch_backend": "html_requests"},
+              "fetch_backend": "html_requests",
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -192,7 +194,8 @@ def test_extra_filters(client, live_server, measure_memory_usage):
               "trim_text_whitespace": "y",
               "sort_text_alphabetically": "",  # leave this OFF for testing
               "url": test_url,
-              "fetch_backend": "html_requests"},
+              "fetch_backend": "html_requests",
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_watch_fields_storage.py

@@ -28,7 +28,8 @@ def test_check_watch_field_storage(client, live_server, measure_memory_usage):
                "url": test_url,
                "tags": "woohoo",
                "headers": "curl:foo",
-               'fetch_backend': "html_requests"
+               'fetch_backend': "html_requests",
+               "time_between_check_use_default": "y"
                },
         follow_redirects=True
     )

changedetectionio/tests/test_xpath_selector.py

@@ -92,7 +92,7 @@ def test_check_xpath_filter_utf8(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -146,7 +146,7 @@ def test_check_xpath_text_function_utf8(client, live_server, measure_memory_usag
     wait_for_all_checks(client)
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -188,7 +188,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server, measure_memo
     # Add our URL to the import page
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"include_filters": xpath_filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": xpath_filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -226,7 +226,7 @@ def test_xpath_validation(client, live_server, measure_memory_usage):
 
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"include_filters": "/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": "/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"is not a valid XPath expression" in res.data
@@ -247,7 +247,7 @@ def test_xpath23_prefix_validation(client, live_server, measure_memory_usage):
 
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"include_filters": "xpath:/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": "xpath:/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"is not a valid XPath expression" in res.data
@@ -298,7 +298,7 @@ def test_xpath1_lxml(client, live_server, measure_memory_usage):
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"include_filters": "xpath1://title/text()", "url": test_url, "tags": "", "headers": "",
-              'fetch_backend': "html_requests"},
+              'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
 
@@ -331,7 +331,7 @@ def test_xpath1_validation(client, live_server, measure_memory_usage):
 
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"include_filters": "xpath1:/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": "xpath1:/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"is not a valid XPath expression" in res.data
@@ -359,7 +359,7 @@ def test_check_with_prefix_include_filters(client, live_server, measure_memory_u
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"include_filters": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tags": "", "headers": "",
-              'fetch_backend': "html_requests"},
+              'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
         follow_redirects=True
     )
 
@@ -413,7 +413,8 @@ def test_various_rules(client, live_server, measure_memory_usage):
                   "url": test_url,
                   "tags": "",
                   "headers": "",
-                  'fetch_backend': "html_requests"},
+                  'fetch_backend': "html_requests",
+                  "time_between_check_use_default": "y"},
             follow_redirects=True
         )
         wait_for_all_checks(client)
@@ -444,7 +445,8 @@ def test_xpath_20(client, live_server, measure_memory_usage):
               "url": test_url,
               "tags": "",
               "headers": "",
-              'fetch_backend': "html_requests"},
+              'fetch_backend': "html_requests",
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
 
@@ -481,7 +483,8 @@ def test_xpath_20_function_count(client, live_server, measure_memory_usage):
               "url": test_url,
               "tags": "",
               "headers": "",
-              'fetch_backend': "html_requests"},
+              'fetch_backend': "html_requests",
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
 
@@ -517,7 +520,8 @@ def test_xpath_20_function_count2(client, live_server, measure_memory_usage):
               "url": test_url,
               "tags": "",
               "headers": "",
-              'fetch_backend': "html_requests"},
+              'fetch_backend': "html_requests",
+              "time_between_check_use_default": "y"},
         follow_redirects=True
     )
 
@@ -554,7 +558,8 @@ def test_xpath_20_function_string_join_matches(client, live_server, measure_memo
             "url": test_url,
             "tags": "",
             "headers": "",
-            'fetch_backend': "html_requests"},
+            'fetch_backend': "html_requests",
+            "time_between_check_use_default": "y"},
         follow_redirects=True
     )
 

changedetectionio/tests/util.py

@@ -6,9 +6,9 @@ from flask import url_for
 import logging
 import time
 
-def set_original_response():
+def set_original_response(extra_title=''):
-    test_return_data = """<html>
+    test_return_data = f"""<html>
-    <head><title>head title</title></head>
+    <head><title>head title{extra_title}</title></head>
     <body>
      Some initial text<br>
      <p>Which is across multiple lines</p>

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -36,6 +36,7 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
             # For now, cookies doesnt work in headers because it must be a full cookiejar object
             'headers': "testheader: yes\buser-agent: MyCustomAgent",
             'fetch_backend': "html_webdriver",
+            "time_between_check_use_default": "y",
         },
         follow_redirects=True
     )
@@ -116,6 +117,7 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
             'browser_steps-1-optional_value': '',
             # For now, cookies doesnt work in headers because it must be a full cookiejar object
             'headers': "testheader: yes\buser-agent: MyCustomAgent",
+            "time_between_check_use_default": "y",
         },
         follow_redirects=True
     )
@@ -167,7 +169,8 @@ def test_non_200_errors_report_browsersteps(client, live_server):
               'fetch_backend': "html_webdriver",
               'browser_steps-0-operation': 'Click element',
               'browser_steps-0-selector': 'button[name=test-button]',
-              'browser_steps-0-optional_value': ''
+              'browser_steps-0-optional_value': '',
+              "time_between_check_use_default": "y"
         },
         follow_redirects=True
     )

changedetectionio/widgets/__init__.py

@@ -0,0 +1,3 @@
+from .ternary_boolean import TernaryNoneBooleanWidget, TernaryNoneBooleanField
+
+__all__ = ['TernaryNoneBooleanWidget', 'TernaryNoneBooleanField']