Re #2826 - Windows was sometimes missing timezone data

.dockerignore

@@ -1,18 +1,31 @@
-.git
+# Git
-.github
+.git/
-changedetectionio/processors/__pycache__
+.gitignore
-changedetectionio/api/__pycache__
-changedetectionio/model/__pycache__
-changedetectionio/blueprint/price_data_follower/__pycache__
-changedetectionio/blueprint/tags/__pycache__
-changedetectionio/blueprint/__pycache__
-changedetectionio/blueprint/browser_steps/__pycache__
-changedetectionio/fetchers/__pycache__
-changedetectionio/tests/visualselector/__pycache__
-changedetectionio/tests/restock/__pycache__
-changedetectionio/tests/__pycache__
-changedetectionio/tests/fetchers/__pycache__
-changedetectionio/tests/unit/__pycache__
-changedetectionio/tests/proxy_list/__pycache__
-changedetectionio/__pycache__
 
+# GitHub
+.github/
+
+# Byte-compiled / optimized / DLL files
+**/__pycache__
+**/*.py[cod]
+
+# Caches
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+
+# Distribution / packaging
+build/
+dist/
+*.egg-info*
+
+# Virtual environment
+.env
+.venv/
+venv/
+
+# IntelliJ IDEA
+.idea/
+
+# Visual Studio
+.vscode/

.github/workflows/test-only.yml

@@ -37,3 +37,10 @@ jobs:
       python-version: '3.12'
       skip-pypuppeteer: true
 
+  test-application-3-13:
+    needs: lint-code
+    uses: ./.github/workflows/test-stack-reusable-workflow.yml
+    with:
+      python-version: '3.13'
+      skip-pypuppeteer: true
+      

.gitignore

@@ -1,15 +1,29 @@
-__pycache__
+# Byte-compiled / optimized / DLL files
-.idea
+**/__pycache__
-*.pyc
+**/*.py[cod]
-datastore/url-watches.json
+
-datastore/*
+# Caches
-__pycache__
+.mypy_cache/
-.pytest_cache
+.pytest_cache/
-build
+.ruff_cache/
-dist
+
-venv
+# Distribution / packaging
-test-datastore/*
+build/
-test-datastore
+dist/
-test-memory.log
 *.egg-info*
+
+# Virtual environment
+.env
+.venv/
+venv/
+
+# IDEs
+.idea
 .vscode/settings.json
+
+# Datastore files
+datastore/
+test-datastore/
+
+# Memory consumption log
+test-memory.log

COMMERCIAL_LICENCE.md

@@ -4,7 +4,7 @@ In any commercial activity involving 'Hosting' (as defined herein), whether in p
 
 # Commercial License Agreement
 
-This Commercial License Agreement ("Agreement") is entered into by and between Mr Morresi (the original creator of this software) here-in ("Licensor") and (your company or personal name) _____________ ("Licensee"). This Agreement sets forth the terms and conditions under which Licensor provides its software ("Software") and services to Licensee for the purpose of reselling the software either in part or full, as part of any commercial activity where the activity involves a third party.
+This Commercial License Agreement ("Agreement") is entered into by and between Web Technologies s.r.o. here-in ("Licensor") and (your company or personal name) _____________ ("Licensee"). This Agreement sets forth the terms and conditions under which Licensor provides its software ("Software") and services to Licensee for the purpose of reselling the software either in part or full, as part of any commercial activity where the activity involves a third party.
 
 ### Definition of Hosting
 

Dockerfile

@@ -32,7 +32,7 @@ RUN pip install --extra-index-url https://www.piwheels.org/simple  --target=/dep
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
 # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
-RUN pip install --target=/dependencies playwright~=1.41.2 \
+RUN pip install --target=/dependencies playwright~=1.48.0 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage

README.md

@@ -105,6 +105,15 @@ We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) glob
 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
+### Schedule web page watches in any timezone, limit by day of week and time.
+
+Easily set a re-check schedule, for example you could limit the web page change detection to only operate during business hours.
+Or perhaps based on a foreign timezone (for example, you want to check for the latest news-headlines in a foreign country at 0900 AM),
+
+<img src="./docs/scheduler.png" style="max-width:80%;" alt="How to monitor web page changes according to a schedule"  title="How to monitor web page changes according to a schedule"  />
+
+Includes quick short-cut buttons to setup a schedule for **business hours only**, or **weekends**.
+
 ### We have a Chrome extension!
 
 Easily add the current web page to your changedetection.io tool, simply install the extension and click "Sync" to connect it to your existing changedetection.io install.

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.47.06'
+__version__ = '0.48.03'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
@@ -160,11 +160,10 @@ def main():
                     )
 
     # Monitored websites will not receive a Referer header when a user clicks on an outgoing link.
-    # @Note: Incompatible with password login (and maybe other features) for now, submit a PR!
     @app.after_request
     def hide_referrer(response):
         if strtobool(os.getenv("HIDE_REFERER", 'false')):
-            response.headers["Referrer-Policy"] = "no-referrer"
+            response.headers["Referrer-Policy"] = "same-origin"
 
         return response
 

changedetectionio/blueprint/tags/__init__.py

@@ -13,6 +13,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def tags_overview_page():
         from .form import SingleTag
         add_form = SingleTag(request.form)
+
         sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
 
         from collections import Counter
@@ -104,9 +105,11 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         default = datastore.data['settings']['application']['tags'].get(uuid)
 
-        form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None,
+        form = group_restock_settings_form(
+                                       formdata=request.form if request.method == 'POST' else None,
                                        data=default,
-                                       extra_notification_tokens=datastore.get_unique_notification_tokens_available()
+                                       extra_notification_tokens=datastore.get_unique_notification_tokens_available(),
+                                       default_system_settings = datastore.data['settings'],
                                        )
 
         template_args = {

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -39,6 +39,7 @@ function isItemInStock() {
         'let me know when it\'s available',
         'mail me when available',
         'message if back in stock',
+        'mevcut değil',
         'nachricht bei',
         'nicht auf lager',
         'nicht lagernd',
@@ -50,7 +51,7 @@ function isItemInStock() {
         'niet beschikbaar',
         'niet leverbaar',
         'niet op voorraad',
-        'no disponible temporalmente',
+        'no disponible',
         'no longer in stock',
         'no tickets available',
         'not available',
@@ -67,12 +68,14 @@ function isItemInStock() {
         'produkt niedostępny',
         'sold out',
         'sold-out',
+        'stokta yok',
         'temporarily out of stock',
         'temporarily unavailable',
         'there were no search results for',
         'this item is currently unavailable',
         'tickets unavailable',
         'tijdelijk uitverkocht',
+        'tükendi',
         'unavailable nearby',
         'unavailable tickets',
         'vergriffen',

changedetectionio/diff.py

@@ -1,6 +1,9 @@
 import difflib
 from typing import List, Iterator, Union
 
+REMOVED_STYLE = "background-color: #fadad7; color: #b30000;"
+ADDED_STYLE = "background-color: #eaf2c2; color: #406619;"
+
 def same_slicer(lst: List[str], start: int, end: int) -> List[str]:
     """Return a slice of the list, or a single element if start == end."""
     return lst[start:end] if start != end else [lst[start]]
@@ -12,11 +15,12 @@ def customSequenceMatcher(
     include_removed: bool = True,
     include_added: bool = True,
     include_replaced: bool = True,
-    include_change_type_prefix: bool = True
+    include_change_type_prefix: bool = True,
+    html_colour: bool = False
 ) -> Iterator[List[str]]:
     """
     Compare two sequences and yield differences based on specified parameters.
-    
+
     Args:
         before (List[str]): Original sequence
         after (List[str]): Modified sequence
@@ -25,26 +29,35 @@ def customSequenceMatcher(
         include_added (bool): Include added parts
         include_replaced (bool): Include replaced parts
         include_change_type_prefix (bool): Add prefixes to indicate change types
-    
+        html_colour (bool): Use HTML background colors for differences
+
     Yields:
         List[str]: Differences between sequences
     """
     cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \t", a=before, b=after)
-    
+
+
+
     for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
         if include_equal and tag == 'equal':
             yield before[alo:ahi]
         elif include_removed and tag == 'delete':
-            prefix = "(removed) " if include_change_type_prefix else ''
+            if html_colour:
-            yield [f"{prefix}{line}" for line in same_slicer(before, alo, ahi)]
+                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)]
+            else:
+                yield [f"(removed) {line}" for line in same_slicer(before, alo, ahi)] if include_change_type_prefix else same_slicer(before, alo, ahi)
         elif include_replaced and tag == 'replace':
-            prefix_changed = "(changed) " if include_change_type_prefix else ''
+            if html_colour:
-            prefix_into = "(into) " if include_change_type_prefix else ''
+                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)] + \
-            yield [f"{prefix_changed}{line}" for line in same_slicer(before, alo, ahi)] + \
+                      [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
-                  [f"{prefix_into}{line}" for line in same_slicer(after, blo, bhi)]
+            else:
+                yield [f"(changed) {line}" for line in same_slicer(before, alo, ahi)] + \
+                      [f"(into) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(before, alo, ahi) + same_slicer(after, blo, bhi)
         elif include_added and tag == 'insert':
-            prefix = "(added) " if include_change_type_prefix else ''
+            if html_colour:
-            yield [f"{prefix}{line}" for line in same_slicer(after, blo, bhi)]
+                yield [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
+            else:
+                yield [f"(added) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(after, blo, bhi)
 
 def render_diff(
     previous_version_file_contents: str,
@@ -55,11 +68,12 @@ def render_diff(
     include_replaced: bool = True,
     line_feed_sep: str = "\n",
     include_change_type_prefix: bool = True,
-    patch_format: bool = False
+    patch_format: bool = False,
+    html_colour: bool = False
 ) -> str:
     """
     Render the difference between two file contents.
-    
+
     Args:
         previous_version_file_contents (str): Original file contents
         newest_version_file_contents (str): Modified file contents
@@ -70,7 +84,8 @@ def render_diff(
         line_feed_sep (str): Separator for lines in output
         include_change_type_prefix (bool): Add prefixes to indicate change types
         patch_format (bool): Use patch format for output
-    
+        html_colour (bool): Use HTML background colors for differences
+
     Returns:
         str: Rendered difference
     """
@@ -88,10 +103,11 @@ def render_diff(
         include_removed=include_removed,
         include_added=include_added,
         include_replaced=include_replaced,
-        include_change_type_prefix=include_change_type_prefix
+        include_change_type_prefix=include_change_type_prefix,
+        html_colour=html_colour
     )
 
     def flatten(lst: List[Union[str, List[str]]]) -> str:
         return line_feed_sep.join(flatten(x) if isinstance(x, list) else x for x in lst)
 
-    return flatten(rendered_diff)
+    return flatten(rendered_diff)

changedetectionio/flask_app.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import datetime
+from zoneinfo import ZoneInfo
 
 import flask_login
 import locale
@@ -42,6 +43,7 @@ from loguru import logger
 from changedetectionio import html_tools, __version__
 from changedetectionio import queuedWatchMetaData
 from changedetectionio.api import api_v1
+from .time_handler import is_within_schedule
 
 datastore = None
 
@@ -53,6 +55,7 @@ extra_stylesheets = []
 
 update_q = queue.PriorityQueue()
 notification_q = queue.Queue()
+MAX_QUEUE_SIZE = 2000
 
 app = Flask(__name__,
             static_url_path="",
@@ -83,7 +86,7 @@ csrf = CSRFProtect()
 csrf.init_app(app)
 notification_debug_log=[]
 
-# get locale ready
+# Locale for correct presentation of prices etc
 default_locale = locale.getdefaultlocale()
 logger.info(f"System locale default is {default_locale}")
 try:
@@ -715,7 +718,8 @@ def changedetection_app(config=None, datastore_o=None):
 
         form = form_class(formdata=request.form if request.method == 'POST' else None,
                           data=default,
-                          extra_notification_tokens=default.extra_notification_token_values()
+                          extra_notification_tokens=default.extra_notification_token_values(),
+                          default_system_settings=datastore.data['settings']
                           )
 
         # For the form widget tag UUID back to "string name" for the field
@@ -803,7 +807,33 @@ def changedetection_app(config=None, datastore_o=None):
             # But in the case something is added we should save straight away
             datastore.needs_write_urgent = True
 
-            if not datastore.data['watching'][uuid].get('paused'):
+            # Do not queue on edit if its not within the time range
+
+            # @todo maybe it should never queue anyway on edit...
+            is_in_schedule = True
+            watch = datastore.data['watching'].get(uuid)
+
+            if watch.get('time_between_check_use_default'):
+                time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {})
+            else:
+                time_schedule_limit = watch.get('time_schedule_limit')
+
+            tz_name = time_schedule_limit.get('timezone')
+            if not tz_name:
+                tz_name =  datastore.data['settings']['application'].get('timezone', 'UTC')
+
+            if time_schedule_limit and time_schedule_limit.get('enabled'):
+                try:
+                    is_in_schedule = is_within_schedule(time_schedule_limit=time_schedule_limit,
+                                                        default_tz=tz_name
+                                                        )
+                except Exception as e:
+                    logger.error(
+                        f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}")
+                    return False
+
+            #############################
+            if not datastore.data['watching'][uuid].get('paused') and is_in_schedule:
                 # Queue the watch for immediate recheck, with a higher priority
                 update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
 
@@ -811,7 +841,7 @@ def changedetection_app(config=None, datastore_o=None):
             if request.args.get("next") and request.args.get("next") == 'diff':
                 return redirect(url_for('diff_history_page', uuid=uuid))
 
-            return redirect(url_for('index'))
+            return redirect(url_for('index', tag=request.args.get("tag",'')))
 
         else:
             if request.method == 'POST' and not form.validate():
@@ -835,15 +865,18 @@ def changedetection_app(config=None, datastore_o=None):
             if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
                 is_html_webdriver = True
 
+            from zoneinfo import available_timezones
+
             # Only works reliably with Playwright
             visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver
             template_args = {
                 'available_processors': processors.available_processors(),
+                'available_timezones': sorted(available_timezones()),
                 'browser_steps_config': browser_step_ui_config,
                 'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
-                'extra_title': f" - Edit - {watch.label}",
-                'extra_processor_config': form.extra_tab_content(),
                 'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
+                'extra_processor_config': form.extra_tab_content(),
+                'extra_title': f" - Edit - {watch.label}",
                 'form': form,
                 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False,
                 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
@@ -852,6 +885,7 @@ def changedetection_app(config=None, datastore_o=None):
                 'jq_support': jq_support,
                 'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False),
                 'settings_application': datastore.data['settings']['application'],
+                'timezone_default_config': datastore.data['settings']['application'].get('timezone'),
                 'using_global_webdriver_wait': not default['webdriver_delay'],
                 'uuid': uuid,
                 'visualselector_enabled': visualselector_enabled,
@@ -881,6 +915,8 @@ def changedetection_app(config=None, datastore_o=None):
     @login_optionally_required
     def settings_page():
         from changedetectionio import forms
+        from datetime import datetime
+        from zoneinfo import available_timezones
 
         default = deepcopy(datastore.data['settings'])
         if datastore.proxy_list is not None:
@@ -948,14 +984,20 @@ def changedetection_app(config=None, datastore_o=None):
             else:
                 flash("An error occurred, please see below.", "error")
 
+        # Convert to ISO 8601 format, all date/time relative events stored as UTC time
+        utc_time = datetime.now(ZoneInfo("UTC")).isoformat()
+
         output = render_template("settings.html",
                                  api_key=datastore.data['settings']['application'].get('api_access_token'),
+                                 available_timezones=sorted(available_timezones()),
                                  emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
                                  extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(),
                                  form=form,
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
                                  min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)),
-                                 settings_application=datastore.data['settings']['application']
+                                 settings_application=datastore.data['settings']['application'],
+                                 timezone_default_config=datastore.data['settings']['application'].get('timezone'),
+                                 utc_time=utc_time,
                                  )
 
         return output
@@ -1268,12 +1310,23 @@ def changedetection_app(config=None, datastore_o=None):
 
             # These files should be in our subdirectory
             try:
-                # set nocache, set content-type
+                # set nocache, set content-type,
-                response = make_response(send_from_directory(os.path.join(datastore_o.datastore_path, filename), "elements.json"))
+                # `filename` is actually directory UUID of the watch
-                response.headers['Content-type'] = 'application/json'
+                watch_directory = str(os.path.join(datastore_o.datastore_path, filename))
-                response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+                response = None
-                response.headers['Pragma'] = 'no-cache'
+                if os.path.isfile(os.path.join(watch_directory, "elements.deflate")):
-                response.headers['Expires'] = 0
+                    response = make_response(send_from_directory(watch_directory, "elements.deflate"))
+                    response.headers['Content-Type'] = 'application/json'
+                    response.headers['Content-Encoding'] = 'deflate'
+                else:
+                    logger.error(f'Request elements.deflate at "{watch_directory}" but was notfound.')
+                    abort(404)
+
+                if response:
+                    response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+                    response.headers['Pragma'] = 'no-cache'
+                    response.headers['Expires'] = "0"
+
                 return response
 
             except FileNotFoundError:
@@ -1342,13 +1395,13 @@ def changedetection_app(config=None, datastore_o=None):
         if new_uuid:
             if add_paused:
                 flash('Watch added in Paused state, saving will unpause.')
-                return redirect(url_for('edit_page', uuid=new_uuid, unpause_on_save=1))
+                return redirect(url_for('edit_page', uuid=new_uuid, unpause_on_save=1, tag=request.args.get('tag')))
             else:
                 # Straight into the queue.
                 update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
                 flash("Watch added.")
 
-        return redirect(url_for('index'))
+        return redirect(url_for('index', tag=request.args.get('tag','')))
 
 
 
@@ -1618,13 +1671,12 @@ def changedetection_app(config=None, datastore_o=None):
     import changedetectionio.blueprint.backups as backups
     app.register_blueprint(backups.construct_blueprint(datastore), url_prefix='/backups')
 
-
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
     threading.Thread(target=notification_runner).start()
 
     # Check for new release version, but not when running in test/build or pytest
-    if not os.getenv("GITHUB_REF", False) and not config.get('disable_checkver') == True:
+    if not os.getenv("GITHUB_REF", False) and not strtobool(os.getenv('DISABLE_VERSION_CHECK', 'no')):
         threading.Thread(target=check_for_new_version).start()
 
     return app
@@ -1708,7 +1760,6 @@ def notification_runner():
 def ticker_thread_check_time_launch_checks():
     import random
     from changedetectionio import update_worker
-
     proxy_last_called_time = {}
 
     recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3))
@@ -1742,12 +1793,14 @@ def ticker_thread_check_time_launch_checks():
             except RuntimeError as e:
                 # RuntimeError: dictionary changed size during iteration
                 time.sleep(0.1)
+                watch_uuid_list = []
             else:
                 break
 
         # Re #438 - Don't place more watches in the queue to be checked if the queue is already large
         while update_q.qsize() >= 2000:
-            time.sleep(1)
+            logger.warning(f"Recheck watches queue size limit reached ({MAX_QUEUE_SIZE}), skipping adding more items")
+            time.sleep(3)
 
 
         recheck_time_system_seconds = int(datastore.threshold_seconds)
@@ -1764,6 +1817,28 @@ def ticker_thread_check_time_launch_checks():
             if watch['paused']:
                 continue
 
+            # @todo - Maybe make this a hook?
+            # Time schedule limit - Decide between watch or global settings
+            if watch.get('time_between_check_use_default'):
+                time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {})
+                logger.trace(f"{uuid} Time scheduler - Using system/global settings")
+            else:
+                time_schedule_limit = watch.get('time_schedule_limit')
+                logger.trace(f"{uuid} Time scheduler - Using watch settings (not global settings)")
+            tz_name = datastore.data['settings']['application'].get('timezone', 'UTC')
+
+            if time_schedule_limit and time_schedule_limit.get('enabled'):
+                try:
+                    result = is_within_schedule(time_schedule_limit=time_schedule_limit,
+                                                default_tz=tz_name
+                                                )
+                    if not result:
+                        logger.trace(f"{uuid} Time scheduler - not within schedule skipping.")
+                        continue
+                except Exception as e:
+                    logger.error(
+                        f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}")
+                    return False
             # If they supplied an individual entry minutes to threshold.
             threshold = recheck_time_system_seconds if watch.get('time_between_check_use_default') else watch.threshold_seconds()
 

changedetectionio/forms.py

@@ -1,12 +1,14 @@
 import os
 import re
 from loguru import logger
+from wtforms.widgets.core import TimeInput
 
 from changedetectionio.strtobool import strtobool
 
 from wtforms import (
     BooleanField,
     Form,
+    Field,
     IntegerField,
     RadioField,
     SelectField,
@@ -125,6 +127,87 @@ class StringTagUUID(StringField):
 
         return 'error'
 
+class TimeDurationForm(Form):
+    hours = SelectField(choices=[(f"{i}", f"{i}") for i in range(0, 25)], default="24",  validators=[validators.Optional()])
+    minutes = SelectField(choices=[(f"{i}", f"{i}") for i in range(0, 60)], default="00", validators=[validators.Optional()])
+
+class TimeStringField(Field):
+    """
+    A WTForms field for time inputs (HH:MM) that stores the value as a string.
+    """
+    widget = TimeInput()  # Use the built-in time input widget
+
+    def _value(self):
+        """
+        Returns the value for rendering in the form.
+        """
+        return self.data if self.data is not None else ""
+
+    def process_formdata(self, valuelist):
+        """
+        Processes the raw input from the form and stores it as a string.
+        """
+        if valuelist:
+            time_str = valuelist[0]
+            # Simple validation for HH:MM format
+            if not time_str or len(time_str.split(":")) != 2:
+                raise ValidationError("Invalid time format. Use HH:MM.")
+            self.data = time_str
+
+
+class validateTimeZoneName(object):
+    """
+       Flask wtform validators wont work with basic auth
+    """
+
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        from zoneinfo import available_timezones
+        python_timezones = available_timezones()
+        if field.data and field.data not in python_timezones:
+            raise ValidationError("Not a valid timezone name")
+
+class ScheduleLimitDaySubForm(Form):
+    enabled = BooleanField("not set", default=True)
+    start_time = TimeStringField("Start At", default="00:00", render_kw={"placeholder": "HH:MM"}, validators=[validators.Optional()])
+    duration = FormField(TimeDurationForm, label="Run duration")
+
+class ScheduleLimitForm(Form):
+    enabled = BooleanField("Use time scheduler", default=False)
+    # Because the label for=""" doesnt line up/work with the actual checkbox
+    monday = FormField(ScheduleLimitDaySubForm, label="")
+    tuesday = FormField(ScheduleLimitDaySubForm, label="")
+    wednesday = FormField(ScheduleLimitDaySubForm, label="")
+    thursday = FormField(ScheduleLimitDaySubForm, label="")
+    friday = FormField(ScheduleLimitDaySubForm, label="")
+    saturday = FormField(ScheduleLimitDaySubForm, label="")
+    sunday = FormField(ScheduleLimitDaySubForm, label="")
+
+    timezone = StringField("Optional timezone to run in",
+                                  render_kw={"list": "timezones"},
+                                  validators=[validateTimeZoneName()]
+                                  )
+    def __init__(
+        self,
+        formdata=None,
+        obj=None,
+        prefix="",
+        data=None,
+        meta=None,
+        **kwargs,
+    ):
+        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
+        self.monday.form.enabled.label.text="Monday"
+        self.tuesday.form.enabled.label.text = "Tuesday"
+        self.wednesday.form.enabled.label.text = "Wednesday"
+        self.thursday.form.enabled.label.text = "Thursday"
+        self.friday.form.enabled.label.text = "Friday"
+        self.saturday.form.enabled.label.text = "Saturday"
+        self.sunday.form.enabled.label.text = "Sunday"
+
+
 class TimeBetweenCheckForm(Form):
     weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
     days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
@@ -225,8 +308,12 @@ class ValidateAppRiseServers(object):
         # so that the custom endpoints are registered
         from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper
         for server_url in field.data:
-            if not apobj.add(server_url):
+            url = server_url.strip()
-                message = field.gettext('\'%s\' is not a valid AppRise URL.' % (server_url))
+            if url.startswith("#"):
+                continue
+
+            if not apobj.add(url):
+                message = field.gettext('\'%s\' is not a valid AppRise URL.' % (url))
                 raise ValidationError(message)
 
 class ValidateJinja2Template(object):
@@ -279,6 +366,7 @@ class validateURL(object):
         # This should raise a ValidationError() or not
         validate_url(field.data)
 
+
 def validate_url(test_url):
     # If hosts that only contain alphanumerics are allowed ("localhost" for example)
     try:
@@ -438,6 +526,7 @@ class commonSettingsForm(Form):
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
     processor = RadioField( label=u"Processor - What do you want to achieve?", choices=processors.available_processors(), default="text_json_diff")
+    timezone = StringField("Timezone for watch schedule", render_kw={"list": "timezones"}, validators=[validateTimeZoneName()])
     webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")])
 
 
@@ -448,7 +537,6 @@ class importForm(Form):
     xlsx_file = FileField('Upload .xlsx file', validators=[FileAllowed(['xlsx'], 'Must be .xlsx file!')])
     file_mapping = SelectField('File mapping', [validators.DataRequired()], choices={('wachete', 'Wachete mapping'), ('custom','Custom mapping')})
 
-
 class SingleBrowserStep(Form):
 
     operation = SelectField('Operation', [validators.Optional()], choices=browser_step_ui_config.keys())
@@ -466,6 +554,9 @@ class processor_text_json_diff_form(commonSettingsForm):
     tags = StringTagUUID('Group tag', [validators.Optional()], default='')
 
     time_between_check = FormField(TimeBetweenCheckForm)
+
+    time_schedule_limit = FormField(ScheduleLimitForm)
+
     time_between_check_use_default = BooleanField('Use global settings for time between check', default=False)
 
     include_filters = StringListField('CSS/JSONPath/JQ/XPath Filters', [ValidateCSSJSONXPATHInput()], default='')
@@ -567,6 +658,23 @@ class processor_text_json_diff_form(commonSettingsForm):
 
         return result
 
+    def __init__(
+            self,
+            formdata=None,
+            obj=None,
+            prefix="",
+            data=None,
+            meta=None,
+            **kwargs,
+    ):
+        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
+        if kwargs and kwargs.get('default_system_settings'):
+            default_tz = kwargs.get('default_system_settings').get('application', {}).get('timezone')
+            if default_tz:
+                self.time_schedule_limit.form.timezone.render_kw['placeholder'] = default_tz
+
+
+
 class SingleExtraProxy(Form):
 
     # maybe better to set some <script>var..
@@ -587,6 +695,7 @@ class DefaultUAInputForm(Form):
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
     time_between_check = FormField(TimeBetweenCheckForm)
+    time_schedule_limit = FormField(ScheduleLimitForm)
     proxy = RadioField('Proxy')
     jitter_seconds = IntegerField('Random jitter seconds ± check',
                                   render_kw={"style": "width: 5em;"},

changedetectionio/model/App.py

@@ -52,7 +52,8 @@ class model(dict):
                     'schema_version' : 0,
                     'shared_diff_access': False,
                     'webdriver_delay': None , # Extra delay in seconds before extracting text
-                    'tags': {} #@todo use Tag.model initialisers
+                    'tags': {}, #@todo use Tag.model initialisers
+                    'timezone': None, # Default IANA timezone name
                 }
             }
         }

changedetectionio/model/Watch.py

@@ -339,7 +339,6 @@ class model(watch_base):
         # @todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status
         return snapshot_fname
 
-    @property
     @property
     def has_empty_checktime(self):
         # using all() + dictionary comprehension
@@ -538,16 +537,17 @@ class model(watch_base):
 
     def save_xpath_data(self, data, as_error=False):
         import json
+        import zlib
 
         if as_error:
-            target_path = os.path.join(self.watch_data_dir, "elements-error.json")
+            target_path = os.path.join(str(self.watch_data_dir), "elements-error.deflate")
         else:
-            target_path = os.path.join(self.watch_data_dir, "elements.json")
+            target_path = os.path.join(str(self.watch_data_dir), "elements.deflate")
 
         self.ensure_data_dir_exists()
 
-        with open(target_path, 'w') as f:
+        with open(target_path, 'wb') as f:
-            f.write(json.dumps(data))
+            f.write(zlib.compress(json.dumps(data).encode()))
             f.close()
 
     # Save as PNG, PNG is larger but better for doing visual diff in the future

changedetectionio/model/__init__.py

@@ -59,6 +59,65 @@ class watch_base(dict):
             'text_should_not_be_present': [],  # Text that should not present
             'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
             'time_between_check_use_default': True,
+            "time_schedule_limit": {
+                "enabled": False,
+                "monday": {
+                    "enabled": True,
+                    "start_time": "00:00",
+                    "duration": {
+                        "hours": "24",
+                        "minutes": "00"
+                    }
+                },
+                "tuesday": {
+                    "enabled": True,
+                    "start_time": "00:00",
+                    "duration": {
+                        "hours": "24",
+                        "minutes": "00"
+                    }
+                },
+                "wednesday": {
+                    "enabled": True,
+                    "start_time": "00:00",
+                    "duration": {
+                        "hours": "24",
+                        "minutes": "00"
+                    }
+                },
+                "thursday": {
+                    "enabled": True,
+                    "start_time": "00:00",
+                    "duration": {
+                        "hours": "24",
+                        "minutes": "00"
+                    }
+                },
+                "friday": {
+                    "enabled": True,
+                    "start_time": "00:00",
+                    "duration": {
+                        "hours": "24",
+                        "minutes": "00"
+                    }
+                },
+                "saturday": {
+                    "enabled": True,
+                    "start_time": "00:00",
+                    "duration": {
+                        "hours": "24",
+                        "minutes": "00"
+                    }
+                },
+                "sunday": {
+                    "enabled": True,
+                    "start_time": "00:00",
+                    "duration": {
+                        "hours": "24",
+                        "minutes": "00"
+                    }
+                },
+            },
             'title': None,
             'track_ldjson_price_data': None,
             'trim_text_whitespace': False,

changedetectionio/notification.py

@@ -23,7 +23,7 @@ valid_tokens = {
 }
 
 default_notification_format_for_watch = 'System default'
-default_notification_format = 'Text'
+default_notification_format = 'HTML Color'
 default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
 default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
 
@@ -31,6 +31,7 @@ valid_notification_formats = {
     'Text': NotifyFormat.TEXT,
     'Markdown': NotifyFormat.MARKDOWN,
     'HTML': NotifyFormat.HTML,
+    'HTML Color': 'htmlcolor',
     # Used only for editing a watch (not for global)
     default_notification_format_for_watch: default_notification_format_for_watch
 }
@@ -76,9 +77,16 @@ def process_notification(n_object, datastore):
 
             # Get the notification body from datastore
             n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
+            if n_object.get('notification_format', '').startswith('HTML'):
+                n_body = n_body.replace("\n", '<br>')
+
             n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
 
             url = url.strip()
+            if url.startswith('#'):
+                logger.trace(f"Skipping commented out notification URL - {url}")
+                continue
+
             if not url:
                 logger.warning(f"Process Notification: skipping empty notification URL.")
                 continue

changedetectionio/static/images/schedule.svg

@@ -0,0 +1,225 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   version="1.1"
+   id="Layer_1"
+   x="0px"
+   y="0px"
+   viewBox="0 0 661.20001 665.40002"
+   xml:space="preserve"
+   width="661.20001"
+   height="665.40002"
+   inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)"
+   sodipodi:docname="schedule.svg"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg"><defs
+   id="defs77" /><sodipodi:namedview
+   id="namedview75"
+   pagecolor="#ffffff"
+   bordercolor="#666666"
+   borderopacity="1.0"
+   inkscape:pageshadow="2"
+   inkscape:pageopacity="0.0"
+   inkscape:pagecheckerboard="0"
+   showgrid="false"
+   fit-margin-top="0"
+   fit-margin-left="0"
+   fit-margin-right="0"
+   fit-margin-bottom="0"
+   inkscape:zoom="1.2458671"
+   inkscape:cx="300.59386"
+   inkscape:cy="332.29869"
+   inkscape:window-width="1920"
+   inkscape:window-height="1051"
+   inkscape:window-x="1920"
+   inkscape:window-y="0"
+   inkscape:window-maximized="1"
+   inkscape:current-layer="g72" /> <style
+   type="text/css"
+   id="style2"> .st0{fill:#FFFFFF;} .st1{fill:#C1272D;} .st2{fill:#991D26;} .st3{fill:#CCCCCC;} .st4{fill:#E6E6E6;} .st5{fill:#F7931E;} .st6{fill:#F2F2F2;} .st7{fill:none;stroke:#999999;stroke-width:17.9763;stroke-linecap:round;stroke-miterlimit:10;} .st8{fill:none;stroke:#333333;stroke-width:8.9882;stroke-linecap:round;stroke-miterlimit:10;} .st9{fill:none;stroke:#C1272D;stroke-width:5.9921;stroke-linecap:round;stroke-miterlimit:10;} .st10{fill:#245F7F;} </style> <g
+   id="g72"
+   transform="translate(-149.4,-147.3)"> <path
+   class="st0"
+   d="M 601.2,699.8 H 205 c -30.7,0 -55.6,-24.9 -55.6,-55.6 V 248 c 0,-30.7 24.9,-55.6 55.6,-55.6 h 396.2 c 30.7,0 55.6,24.9 55.6,55.6 v 396.2 c 0,30.7 -24.9,55.6 -55.6,55.6 z"
+   id="path4"
+   style="fill:#dfdfdf;fill-opacity:1" /> <path
+   class="st1"
+   d="M 601.2,192.4 H 205 c -30.7,0 -55.6,24.9 -55.6,55.6 v 88.5 H 656.8 V 248 c 0,-30.7 -24.9,-55.6 -55.6,-55.6 z"
+   id="path6"
+   style="fill:#d62128;fill-opacity:1" /> <circle
+   class="st2"
+   cx="253.3"
+   cy="264.5"
+   r="36.700001"
+   id="circle8" /> <circle
+   class="st2"
+   cx="551.59998"
+   cy="264.5"
+   r="36.700001"
+   id="circle10" /> <path
+   class="st3"
+   d="m 253.3,275.7 v 0 c -11.8,0 -21.3,-9.6 -21.3,-21.3 v -85.8 c 0,-11.8 9.6,-21.3 21.3,-21.3 v 0 c 11.8,0 21.3,9.6 21.3,21.3 v 85.8 c 0,11.8 -9.5,21.3 -21.3,21.3 z"
+   id="path12" /> <path
+   class="st3"
+   d="m 551.6,275.7 v 0 c -11.8,0 -21.3,-9.6 -21.3,-21.3 v -85.8 c 0,-11.8 9.6,-21.3 21.3,-21.3 v 0 c 11.8,0 21.3,9.6 21.3,21.3 v 85.8 c 0.1,11.8 -9.5,21.3 -21.3,21.3 z"
+   id="path14" /> <rect
+   x="215.7"
+   y="370.89999"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect16"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="313"
+   y="370.89999"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect18"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="410.20001"
+   y="370.89999"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect20"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="507.5"
+   y="370.89999"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect22"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="215.7"
+   y="465"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect24"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="313"
+   y="465"
+   class="st1"
+   width="75.199997"
+   height="75.199997"
+   id="rect26"
+   style="fill:#27c12b;fill-opacity:1" /> <rect
+   x="410.20001"
+   y="465"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect28"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="507.5"
+   y="465"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect30" /> <rect
+   x="215.7"
+   y="559.09998"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect32"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="313"
+   y="559.09998"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect34"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="410.20001"
+   y="559.09998"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect36"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="507.5"
+   y="559.09998"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect38" /> <g
+   id="g70"> <circle
+   class="st5"
+   cx="621.90002"
+   cy="624"
+   r="188.7"
+   id="circle40" /> <circle
+   class="st0"
+   cx="621.90002"
+   cy="624"
+   r="148"
+   id="circle42" /> <path
+   class="st6"
+   d="m 486.6,636.8 c 0,-81.7 66.3,-148 148,-148 37.6,0 72,14.1 98.1,37.2 -27.1,-30.6 -66.7,-49.9 -110.8,-49.9 -81.7,0 -148,66.3 -148,148 0,44.1 19.3,83.7 49.9,110.8 -23.1,-26.2 -37.2,-60.5 -37.2,-98.1 z"
+   id="path44" /> <polyline
+   class="st7"
+   points="621.9,530.4 621.9,624 559,624  "
+   id="polyline46" /> <g
+   id="g64"> <line
+   class="st8"
+   x1="621.90002"
+   y1="508.29999"
+   x2="621.90002"
+   y2="497.10001"
+   id="line48" /> <line
+   class="st8"
+   x1="621.90002"
+   y1="756.29999"
+   x2="621.90002"
+   y2="745.09998"
+   id="line50" /> <line
+   class="st8"
+   x1="740.29999"
+   y1="626.70001"
+   x2="751.5"
+   y2="626.70001"
+   id="line52" /> <line
+   class="st8"
+   x1="492.29999"
+   y1="626.70001"
+   x2="503.5"
+   y2="626.70001"
+   id="line54" /> <line
+   class="st8"
+   x1="705.59998"
+   y1="710.40002"
+   x2="713.5"
+   y2="718.29999"
+   id="line56" /> <line
+   class="st8"
+   x1="530.29999"
+   y1="535.09998"
+   x2="538.20001"
+   y2="543"
+   id="line58" /> <line
+   class="st8"
+   x1="538.20001"
+   y1="710.40002"
+   x2="530.29999"
+   y2="718.29999"
+   id="line60" /> <line
+   class="st8"
+   x1="713.5"
+   y1="535.09998"
+   x2="705.59998"
+   y2="543"
+   id="line62" /> </g> <line
+   class="st9"
+   x1="604.40002"
+   y1="606.29999"
+   x2="684.5"
+   y2="687.40002"
+   id="line66" /> <circle
+   class="st10"
+   cx="621.90002"
+   cy="624"
+   r="16.1"
+   id="circle68" /> </g> </g> </svg>

changedetectionio/static/js/global-settings.js

@@ -24,5 +24,19 @@ $(document).ready(function () {
         $(target).toggle();
     });
 
+    // Time zone config related
+    $(".local-time").each(function (e) {
+        $(this).text(new Date($(this).data("utc")).toLocaleString());
+    })
+
+    const timezoneInput = $('#application-timezone');
+    if(timezoneInput.length) {
+        const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+        if (!timezoneInput.val().trim()) {
+            timezoneInput.val(timezone);
+            timezoneInput.after('<div class="timezone-message">The timezone was set from your browser, <strong>be sure to press save!</strong></div>');
+        }
+    }
+
 });
 

changedetectionio/static/js/plugins.js

@@ -159,4 +159,38 @@
         // Return the current request in case it's needed
         return requests[namespace];
     };
-})(jQuery);
+})(jQuery);
+
+
+
+function toggleOpacity(checkboxSelector, fieldSelector, inverted) {
+    const checkbox = document.querySelector(checkboxSelector);
+    const fields = document.querySelectorAll(fieldSelector);
+
+    function updateOpacity() {
+        const opacityValue = !checkbox.checked ? (inverted ? 0.6 : 1) : (inverted ? 1 : 0.6);
+        fields.forEach(field => {
+            field.style.opacity = opacityValue;
+        });
+    }
+
+    // Initial setup
+    updateOpacity();
+    checkbox.addEventListener('change', updateOpacity);
+}
+
+function toggleVisibility(checkboxSelector, fieldSelector, inverted) {
+    const checkbox = document.querySelector(checkboxSelector);
+    const fields = document.querySelectorAll(fieldSelector);
+
+    function updateOpacity() {
+        const opacityValue = !checkbox.checked ? (inverted ? 'none' : 'block') : (inverted ? 'block' : 'none');
+        fields.forEach(field => {
+            field.style.display = opacityValue;
+        });
+    }
+
+    // Initial setup
+    updateOpacity();
+    checkbox.addEventListener('change', updateOpacity);
+}

changedetectionio/static/js/scheduler.js

@@ -0,0 +1,109 @@
+function getTimeInTimezone(timezone) {
+    const now = new Date();
+    const options = {
+        timeZone: timezone,
+        weekday: 'long',
+        year: 'numeric',
+        hour12: false,
+        month: '2-digit',
+        day: '2-digit',
+        hour: '2-digit',
+        minute: '2-digit',
+        second: '2-digit',
+    };
+
+    const formatter = new Intl.DateTimeFormat('en-US', options);
+    return formatter.format(now);
+}
+
+$(document).ready(function () {
+
+    let exceedsLimit = false;
+    const warning_text = $("#timespan-warning")
+    const timezone_text_widget = $("input[id*='time_schedule_limit-timezone']")
+
+    toggleVisibility('#time_schedule_limit-enabled, #requests-time_schedule_limit-enabled', '#schedule-day-limits-wrapper', true)
+
+    setInterval(() => {
+        let success = true;
+        try {
+            // Show the current local time according to either placeholder or entered TZ name
+            if (timezone_text_widget.val().length) {
+                $('#local-time-in-tz').text(getTimeInTimezone(timezone_text_widget.val()));
+            } else {
+                // So maybe use what is in the placeholder (which will be the default settings)
+                $('#local-time-in-tz').text(getTimeInTimezone(timezone_text_widget.attr('placeholder')));
+            }
+        } catch (error) {
+            success = false;
+            $('#local-time-in-tz').text("");
+            console.error(timezone_text_widget.val())
+        }
+
+        $(timezone_text_widget).toggleClass('error', !success);
+
+    }, 500);
+
+    $('#schedule-day-limits-wrapper').on('change click blur', 'input, checkbox, select', function() {
+
+        let allOk = true;
+
+        // Controls setting the warning that the time could overlap into the next day
+        $("li.day-schedule").each(function () {
+            const $schedule = $(this);
+            const $checkbox = $schedule.find("input[type='checkbox']");
+
+            if ($checkbox.is(":checked")) {
+                const timeValue = $schedule.find("input[type='time']").val();
+                const durationHours = parseInt($schedule.find("select[name*='-duration-hours']").val(), 10) || 0;
+                const durationMinutes = parseInt($schedule.find("select[name*='-duration-minutes']").val(), 10) || 0;
+
+                if (timeValue) {
+                    const [startHours, startMinutes] = timeValue.split(":").map(Number);
+                    const totalMinutes = (startHours * 60 + startMinutes) + (durationHours * 60 + durationMinutes);
+
+                    exceedsLimit = totalMinutes > 1440
+                    if (exceedsLimit) {
+                        allOk = false
+                    }
+                    // Set the row/day-of-week highlight
+                    $schedule.toggleClass("warning", exceedsLimit);
+                }
+            } else {
+                $schedule.toggleClass("warning", false);
+            }
+        });
+
+        warning_text.toggle(!allOk)
+    });
+
+    $('table[id*="time_schedule_limit-saturday"], table[id*="time_schedule_limit-sunday"]').addClass("weekend-day")
+
+    // Presets [weekend] [business hours] etc
+    $(document).on('click', '[data-template].set-schedule', function () {
+        // Get the value of the 'data-template' attribute
+        switch ($(this).attr('data-template')) {
+            case 'business-hours':
+                $('.day-schedule table:not(.weekend-day) input[type="time"]').val('09:00')
+                $('.day-schedule table:not(.weekend-day) select[id*="-duration-hours"]').val('8');
+                $('.day-schedule table:not(.weekend-day) select[id*="-duration-minutes"]').val('0');
+                $('.day-schedule input[id*="-enabled"]').prop('checked', true);
+                $('.day-schedule .weekend-day input[id*="-enabled"]').prop('checked', false);
+                break;
+            case 'weekend':
+                $('.day-schedule .weekend-day input[type="time"][id$="start-time"]').val('00:00')
+                $('.day-schedule .weekend-day select[id*="-duration-hours"]').val('24');
+                $('.day-schedule .weekend-day select[id*="-duration-minutes"]').val('0');
+                $('.day-schedule input[id*="-enabled"]').prop('checked', false);
+                $('.day-schedule .weekend-day input[id*="-enabled"]').prop('checked', true);
+                break;
+            case 'reset':
+
+                $('.day-schedule input[type="time"]').val('00:00')
+                $('.day-schedule select[id*="-duration-hours"]').val('24');
+                $('.day-schedule select[id*="-duration-minutes"]').val('0');
+                $('.day-schedule input[id*="-enabled"]').prop('checked', true);
+                break;
+        }
+    });
+});

changedetectionio/static/js/visual-selector.js

@@ -132,6 +132,7 @@ $(document).ready(() => {
         }).done((data) => {
             $fetchingUpdateNoticeElem.html("Rendering..");
             selectorData = data;
+
             sortScrapedElementsBySize();
             console.log(`Reported browser width from backend: ${data['browser_width']}`);
 

changedetectionio/static/js/watch-settings.js

@@ -1,17 +1,3 @@
-function toggleOpacity(checkboxSelector, fieldSelector, inverted) {
-    const checkbox = document.querySelector(checkboxSelector);
-    const fields = document.querySelectorAll(fieldSelector);
-    function updateOpacity() {
-        const opacityValue = !checkbox.checked ? (inverted ? 0.6 : 1) : (inverted ? 1 : 0.6);
-        fields.forEach(field => {
-            field.style.opacity = opacityValue;
-        });
-    }
-    // Initial setup
-    updateOpacity();
-    checkbox.addEventListener('change', updateOpacity);
-}
-
 
 function request_textpreview_update() {
     if (!$('body').hasClass('preview-text-enabled')) {
@@ -57,7 +43,9 @@ function request_textpreview_update() {
     })
 }
 
+
 $(document).ready(function () {
+
     $('#notification-setting-reset-to-default').click(function (e) {
         $('#notification_title').val('');
         $('#notification_body').val('');
@@ -70,11 +58,12 @@ $(document).ready(function () {
         $('#notification-tokens-info').toggle();
     });
 
-    toggleOpacity('#time_between_check_use_default', '#time_between_check', false);
+    toggleOpacity('#time_between_check_use_default', '#time-check-widget-wrapper, #time-between-check-schedule', false);
+
 
     const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
-    $("#text-preview-inner").css('max-height', (vh-300)+"px");
+    $("#text-preview-inner").css('max-height', (vh - 300) + "px");
-    $("#text-preview-before-inner").css('max-height', (vh-300)+"px");
+    $("#text-preview-before-inner").css('max-height', (vh - 300) + "px");
 
     $("#activate-text-preview").click(function (e) {
         $('body').toggleClass('preview-text-enabled')

changedetectionio/store.py

@@ -374,7 +374,7 @@ class ChangeDetectionStore:
     def visualselector_data_is_ready(self, watch_uuid):
         output_path = "{}/{}".format(self.datastore_path, watch_uuid)
         screenshot_filename = "{}/last-screenshot.png".format(output_path)
-        elements_index_filename = "{}/elements.json".format(output_path)
+        elements_index_filename = "{}/elements.deflate".format(output_path)
         if path.isfile(screenshot_filename) and  path.isfile(elements_index_filename) :
             return True
 
@@ -909,3 +909,18 @@ class ChangeDetectionStore:
             if self.data['watching'][uuid].get('in_stock_only'):
                 del (self.data['watching'][uuid]['in_stock_only'])
 
+    # Compress old elements.json to elements.deflate, saving disk, this compression is pretty fast.
+    def update_19(self):
+        import zlib
+
+        for uuid, watch in self.data['watching'].items():
+            json_path = os.path.join(self.datastore_path, uuid, "elements.json")
+            deflate_path = os.path.join(self.datastore_path, uuid, "elements.deflate")
+
+            if os.path.exists(json_path):
+                with open(json_path, "rb") as f_j:
+                    with open(deflate_path, "wb") as f_d:
+                        logger.debug(f"Compressing {str(json_path)} to {str(deflate_path)}..")
+                        f_d.write(zlib.compress(f_j.read()))
+                        os.unlink(json_path)
+

changedetectionio/templates/_helpers.html

@@ -59,4 +59,100 @@
 
 {% macro render_button(field) %}
   {{ field(**kwargs)|safe }}
+{% endmacro %}
+
+{% macro render_time_schedule_form(form, available_timezones, timezone_default_config) %}
+    <style>
+    .day-schedule *, .day-schedule select {
+        display: inline-block;
+    }
+
+    .day-schedule label[for*="time_schedule_limit-"][for$="-enabled"] {
+        min-width: 6rem;
+        font-weight: bold;
+    }
+    .day-schedule label {
+        font-weight: normal;
+    }
+
+    .day-schedule table label {
+        padding-left: 0.5rem;
+        padding-right: 0.5rem;
+    }
+    #timespan-warning, input[id*='time_schedule_limit-timezone'].error {
+        color: #ff0000;
+    }
+    .day-schedule.warning table {
+        background-color: #ffbbc2;
+    }
+    ul#day-wrapper {
+        list-style: none;
+    }
+    #timezone-info > * {
+        display: inline-block;
+    }
+
+    #scheduler-icon-label {
+        background-position: left center;
+        background-repeat: no-repeat;
+        background-size: contain;
+        display: inline-block;
+        vertical-align: middle;
+        padding-left: 50px;
+        background-image: url({{ url_for('static_content', group='images', filename='schedule.svg') }});
+    }
+    #timespan-warning {
+        display: none;
+    }
+    </style>
+    <br>
+
+    {% if timezone_default_config %}
+    <div>
+        <span id="scheduler-icon-label" style="">
+            {{ render_checkbox_field(form.time_schedule_limit.enabled) }}
+            <div class="pure-form-message-inline">
+                Set a hourly/week day schedule
+            </div>
+        </span>
+
+    </div>
+    <br>
+    <div id="schedule-day-limits-wrapper">
+        <label>Schedule time limits</label><a data-template="business-hours"
+                                              class="set-schedule pure-button button-secondary button-xsmall">Business
+        hours</a>
+        <a data-template="weekend" class="set-schedule pure-button button-secondary button-xsmall">Weekends</a>
+        <a data-template="reset" class="set-schedule pure-button button-xsmall">Reset</a><br>
+        <br>
+
+        <ul id="day-wrapper">
+            {% for day in ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] %}
+                <li class="day-schedule" id="schedule-{{ day }}">
+                    {{ render_nolabel_field(form.time_schedule_limit[day]) }}
+                </li>
+            {% endfor %}
+            <li id="timespan-warning">Warning, one or more of your 'days' has a duration that would extend into the next day.<br>
+            This could have unintended consequences.</li>
+            <li id="timezone-info">
+                {{ render_field(form.time_schedule_limit.timezone, placeholder=timezone_default_config) }} <span id="local-time-in-tz"></span>
+                <datalist id="timezones" style="display: none;">
+                    {% for timezone in available_timezones %}
+                        <option value="{{ timezone }}">{{ timezone }}</option>
+                    {% endfor %}
+                </datalist>
+            </li>
+        </ul>
+    <br>
+        <span class="pure-form-message-inline">
+         <a href="https://changedetection.io/tutorials">More help and examples about using the scheduler</a>
+        </span>
+    </div>
+    {% else %}
+        <span class="pure-form-message-inline">
+            Want to use a time schedule? <a href="{{url_for('settings_page')}}#timedate">First confirm/save your Time Zone Settings</a>
+        </span>
+        <br>
+    {% endif %}
+
 {% endmacro %}

changedetectionio/templates/edit.html

@@ -1,10 +1,11 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='scheduler.js')}}" defer></script>
 <script>
     const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
@@ -58,7 +59,7 @@
 
     <div class="box-wrap inner">
         <form class="pure-form pure-form-stacked"
-              action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next'), unpause_on_save = request.args.get('unpause_on_save')) }}" method="POST">
+              action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next'), unpause_on_save = request.args.get('unpause_on_save'), tag = request.args.get('tag')) }}" method="POST">
              <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
 
             <div class="tab-pane-inner" id="general">
@@ -79,9 +80,24 @@
                         <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
                     </div>
                     <div class="pure-control-group time-between-check border-fieldset">
-                        {{ render_field(form.time_between_check, class="time-check-widget") }}
+
                         {{ render_checkbox_field(form.time_between_check_use_default, class="use-default-timecheck") }}
-                    </div>
+                        <br>
+                        <div id="time-check-widget-wrapper">
+                            {{ render_field(form.time_between_check, class="time-check-widget") }}
+
+                            <span class="pure-form-message-inline">
+                             The interval/amount of time between each check.
+                            </span>
+                        </div>
+                        <div id="time-between-check-schedule">
+                            <!-- Start Time and End Time -->
+                            <div id="limit-between-time">
+                                {{ render_time_schedule_form(form, available_timezones, timezone_default_config) }}
+                            </div>
+                        </div>
+<br>
+              </div>
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.extract_title_as_title) }}
                     </div>

changedetectionio/templates/settings.html

@@ -1,7 +1,7 @@
 {% extends 'base.html' %}
 
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script>
     const notification_base_url="{{url_for('ajax_callback_send_notification_test', mode="global-settings")}}";
@@ -10,9 +10,11 @@
 {% endif %}
 </script>
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='plugins.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='scheduler.js')}}" defer></script>
 <div class="edit-form">
     <div class="tabs collapsable">
         <ul>
@@ -21,6 +23,7 @@
             <li class="tab"><a href="#fetching">Fetching</a></li>
             <li class="tab"><a href="#filters">Global Filters</a></li>
             <li class="tab"><a href="#api">API</a></li>
+            <li class="tab"><a href="#timedate">Time &amp Date</a></li>
             <li class="tab"><a href="#proxies">CAPTCHA &amp; Proxies</a></li>
         </ul>
     </div>
@@ -32,6 +35,12 @@
                     <div class="pure-control-group">
                         {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }}
                         <span class="pure-form-message-inline">Default recheck time for all watches, current system minimum is <i>{{min_system_recheck_seconds}}</i> seconds (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Misc-system-settings#enviroment-variables">more info</a>).</span>
+                            <div id="time-between-check-schedule">
+                                <!-- Start Time and End Time -->
+                                <div id="limit-between-time">
+                                    {{ render_time_schedule_form(form.requests, available_timezones, timezone_default_config) }}
+                                </div>
+                        </div>
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
@@ -211,6 +220,23 @@ nav
                     </p>
                 </div>
             </div>
+            <div class="tab-pane-inner" id="timedate">
+                <div class="pure-control-group">
+                    Ensure the settings below are correct, they are used to manage the time schedule for checking your web page watches.
+                </div>
+                <div class="pure-control-group">
+                    <p><strong>UTC Time &amp Date from Server:</strong> <span id="utc-time" >{{ utc_time }}</span></p>
+                    <p><strong>Local Time &amp Date in Browser:</strong> <span class="local-time" data-utc="{{ utc_time }}"></span></p>
+                    <p>
+                       {{ render_field(form.application.form.timezone) }}
+                        <datalist id="timezones" style="display: none;">
+                            {% for tz_name in available_timezones %}
+                                <option value="{{ tz_name }}">{{ tz_name }}</option>
+                            {% endfor %}
+                        </datalist>
+                    </p>
+                </div>
+            </div>
             <div class="tab-pane-inner" id="proxies">
                 <div id="recommended-proxy">
                     <div>

changedetectionio/templates/watch-overview.html

@@ -6,7 +6,7 @@
 
 <div class="box">
 
-    <form class="pure-form" action="{{ url_for('form_quick_watch_add') }}" method="POST" id="new-watch-form">
+    <form class="pure-form" action="{{ url_for('form_quick_watch_add', tag=active_tag_uuid) }}" method="POST" id="new-watch-form">
         <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
         <fieldset>
             <legend>Add a new change detection watch</legend>
@@ -187,7 +187,7 @@
                 <td>
                     <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                        class="recheck pure-button pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
-                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
+                    <a href="{{ url_for('edit_page', uuid=watch.uuid, tag=active_tag_uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
                     {% if watch.history_n >= 2 %}
 
                         {%  if is_unviewed %}

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -113,7 +113,8 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     res = client.post(
         url_for("settings_page"),
         data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
-              "application-notification_body": 'triggered text was -{{triggered_text}}- 网站监测 内容更新了',
+              # triggered_text will contain multiple lines
+              "application-notification_body": 'triggered text was -{{triggered_text}}- ### 网站监测 内容更新了 ####',
               # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
               "application-notification_urls": test_notification_url,
               "application-minutes_between_check": 180,
@@ -171,7 +172,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     assert os.path.isfile("test-datastore/notification.txt"), "Notification fired because I can see the output file"
     with open("test-datastore/notification.txt", 'rb') as f:
         response = f.read()
-        assert b'-Oh yes please-' in response
+        assert b'-Oh yes please' in response
         assert '网站监测 内容更新了'.encode('utf-8') in response
 
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_css_selector.py

@@ -125,8 +125,7 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
 
 # Tests the whole stack works with the CSS Filter
 def test_check_multiple_filters(client, live_server, measure_memory_usage):
-    sleep_time_for_fetch_thread = 3
+    #live_server_setup(live_server)
-
     include_filters = "#blob-a\r\nxpath://*[contains(@id,'blob-b')]"
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -138,9 +137,6 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage):
      </html>
     """)
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -149,7 +145,7 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(1)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -165,7 +161,7 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     res = client.get(
         url_for("preview_page", uuid="first"),

changedetectionio/tests/test_notification.py

@@ -442,4 +442,67 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert b"Error: You must have atleast one watch configured for 'test notification' to work" in res.data
 
 
+def _test_color_notifications(client, notification_body_token):
 
+    from changedetectionio.diff import ADDED_STYLE, REMOVED_STYLE
+
+    set_original_response()
+
+    if os.path.isfile("test-datastore/notification.txt"):
+        os.unlink("test-datastore/notification.txt")
+
+
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
+
+
+    # otherwise other settings would have already existed from previous tests in this file
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "application-fetch_backend": "html_requests",
+            "application-minutes_between_check": 180,
+            "application-notification_body": notification_body_token,
+            "application-notification_format": "HTML Color",
+            "application-notification_urls": test_notification_url,
+            "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
+        },
+        follow_redirects=True
+    )
+    assert b'Settings updated' in res.data
+
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": 'nice one'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    wait_for_all_checks(client)
+
+    set_modified_response()
+
+
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    assert b'1 watches queued for rechecking.' in res.data
+
+    wait_for_all_checks(client)
+    time.sleep(3)
+
+    with open("test-datastore/notification.txt", 'r') as f:
+        x = f.read()
+        assert f'<span style="{REMOVED_STYLE}">Which is across multiple lines' in x
+
+
+    client.get(
+        url_for("form_delete", uuid="all"),
+        follow_redirects=True
+    )
+
+def test_html_color_notifications(client, live_server, measure_memory_usage):
+
+    #live_server_setup(live_server)
+    _test_color_notifications(client, '{{diff}}')
+    _test_color_notifications(client, '{{diff_full}}')
+    

changedetectionio/tests/test_scheduler.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+
+import time
+from datetime import datetime, timezone
+from zoneinfo import ZoneInfo
+from flask import url_for
+from .util import  live_server_setup, wait_for_all_checks, extract_UUID_from_client
+
+def test_setup(client, live_server):
+    live_server_setup(live_server)
+
+def test_check_basic_scheduler_functionality(client, live_server, measure_memory_usage):
+    #live_server_setup(live_server)
+    days = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
+    test_url = url_for('test_random_content_endpoint', _external=True)
+
+    # We use "Pacific/Kiritimati" because its the furthest +14 hours, so it might show up more interesting bugs
+    # The rest of the actual functionality should be covered in the unit-test  unit/test_scheduler.py
+    #####################
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-empty_pages_are_a_change": "",
+              "requests-time_between_check-seconds": 1,
+              "application-timezone": "Pacific/Kiritimati",  # Most Forward Time Zone (UTC+14:00)
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    res = client.get(url_for("settings_page"))
+    assert b'Pacific/Kiritimati' in res.data
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+    uuid = extract_UUID_from_client(client)
+
+    # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
+
+    tpl = {
+        "time_schedule_limit-XXX-start_time": "00:00",
+        "time_schedule_limit-XXX-duration-hours": 24,
+        "time_schedule_limit-XXX-duration-minutes": 0,
+        "time_schedule_limit-XXX-enabled": '',  # All days are turned off
+        "time_schedule_limit-enabled": 'y',  # Scheduler is enabled, all days however are off.
+    }
+
+    scheduler_data = {}
+    for day in days:
+        for key, value in tpl.items():
+            # Replace "XXX" with the current day in the key
+            new_key = key.replace("XXX", day)
+            scheduler_data[new_key] = value
+
+    last_check = live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked']
+    data = {
+        "url": test_url,
+        "fetch_backend": "html_requests"
+    }
+    data.update(scheduler_data)
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data=data,
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    res = client.get(url_for("edit_page", uuid="first"))
+    assert b"Pacific/Kiritimati" in res.data, "Should be Pacific/Kiritimati in placeholder data"
+
+    # "Edit" should not trigger a check because it's not enabled in the schedule.
+    time.sleep(2)
+    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] == last_check
+
+    # Enabling today in Kiritimati should work flawless
+    kiritimati_time = datetime.now(timezone.utc).astimezone(ZoneInfo("Pacific/Kiritimati"))
+    kiritimati_time_day_of_week = kiritimati_time.strftime("%A").lower()
+    live_server.app.config['DATASTORE'].data['watching'][uuid]["time_schedule_limit"][kiritimati_time_day_of_week]["enabled"] = True
+    time.sleep(3)
+    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] != last_check
+
+    # Cleanup everything
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+
+def test_check_basic_global_scheduler_functionality(client, live_server, measure_memory_usage):
+    #live_server_setup(live_server)
+    days = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
+    test_url = url_for('test_random_content_endpoint', _external=True)
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+    uuid = extract_UUID_from_client(client)
+
+    # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
+
+    tpl = {
+        "requests-time_schedule_limit-XXX-start_time": "00:00",
+        "requests-time_schedule_limit-XXX-duration-hours": 24,
+        "requests-time_schedule_limit-XXX-duration-minutes": 0,
+        "requests-time_schedule_limit-XXX-enabled": '',  # All days are turned off
+        "requests-time_schedule_limit-enabled": 'y',  # Scheduler is enabled, all days however are off.
+    }
+
+    scheduler_data = {}
+    for day in days:
+        for key, value in tpl.items():
+            # Replace "XXX" with the current day in the key
+            new_key = key.replace("XXX", day)
+            scheduler_data[new_key] = value
+
+    data = {
+        "application-empty_pages_are_a_change": "",
+        "application-timezone": "Pacific/Kiritimati",  # Most Forward Time Zone (UTC+14:00)
+        'application-fetch_backend': "html_requests",
+        "requests-time_between_check-hours": 0,
+        "requests-time_between_check-minutes": 0,
+        "requests-time_between_check-seconds": 1,
+    }
+    data.update(scheduler_data)
+
+    #####################
+    res = client.post(
+        url_for("settings_page"),
+        data=data,
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    res = client.get(url_for("settings_page"))
+    assert b'Pacific/Kiritimati' in res.data
+
+    wait_for_all_checks(client)
+
+    # UI Sanity check
+
+    res = client.get(url_for("edit_page", uuid="first"))
+    assert b"Pacific/Kiritimati" in res.data, "Should be Pacific/Kiritimati in placeholder data"
+
+    #### HITTING SAVE SHOULD NOT TRIGGER A CHECK
+    last_check = live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked']
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "url": test_url,
+            "fetch_backend": "html_requests",
+            "time_between_check_use_default": "y"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+    time.sleep(2)
+    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] == last_check
+
+    # Enabling "today" in Kiritimati time should make the system check that watch
+    kiritimati_time = datetime.now(timezone.utc).astimezone(ZoneInfo("Pacific/Kiritimati"))
+    kiritimati_time_day_of_week = kiritimati_time.strftime("%A").lower()
+    live_server.app.config['DATASTORE'].data['settings']['requests']['time_schedule_limit'][kiritimati_time_day_of_week]["enabled"] = True
+
+    time.sleep(3)
+    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] != last_check
+
+    # Cleanup everything
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

changedetectionio/tests/unit/test_scheduler.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+# run from dir above changedetectionio/ dir
+# python3 -m unittest changedetectionio.tests.unit.test_jinja2_security
+
+import unittest
+from datetime import datetime, timedelta
+from zoneinfo import ZoneInfo
+
+class TestScheduler(unittest.TestCase):
+
+    # UTC+14:00 (Line Islands, Kiribati) is the farthest ahead, always ahead of UTC.
+    # UTC-12:00 (Baker Island, Howland Island) is the farthest behind, always one calendar day behind UTC.
+
+    def test_timezone_basic_time_within_schedule(self):
+        from changedetectionio import time_handler
+
+        timezone_str = 'Europe/Berlin'
+        debug_datetime = datetime.now(ZoneInfo(timezone_str))
+        day_of_week = debug_datetime.strftime('%A')
+        time_str = str(debug_datetime.hour)+':00'
+        duration = 60  # minutes
+
+        # The current time should always be within 60 minutes of [time_hour]:00
+        result = time_handler.am_i_inside_time(day_of_week=day_of_week,
+                                               time_str=time_str,
+                                               timezone_str=timezone_str,
+                                               duration=duration)
+
+        self.assertEqual(result, True, f"{debug_datetime} is within time scheduler {day_of_week} {time_str} in {timezone_str} for {duration} minutes")
+
+    def test_timezone_basic_time_outside_schedule(self):
+        from changedetectionio import time_handler
+
+        timezone_str = 'Europe/Berlin'
+        # We try a date in the future..
+        debug_datetime = datetime.now(ZoneInfo(timezone_str))+ timedelta(days=-1)
+        day_of_week = debug_datetime.strftime('%A')
+        time_str = str(debug_datetime.hour) + ':00'
+        duration = 60*24  # minutes
+
+        # The current time should always be within 60 minutes of [time_hour]:00
+        result = time_handler.am_i_inside_time(day_of_week=day_of_week,
+                                               time_str=time_str,
+                                               timezone_str=timezone_str,
+                                               duration=duration)
+
+        self.assertNotEqual(result, True,
+                         f"{debug_datetime} is NOT within time scheduler {day_of_week} {time_str} in {timezone_str} for {duration} minutes")
+
+
+if __name__ == '__main__':
+    unittest.main()

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -54,15 +54,21 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
 
 
     assert os.path.isfile(os.path.join('test-datastore', uuid, 'last-screenshot.png')), "last-screenshot.png should exist"
-    assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.json')), "xpath elements.json data should exist"
+    assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.deflate')), "xpath elements.deflate data should exist"
 
     # Open it and see if it roughly looks correct
-    with open(os.path.join('test-datastore', uuid, 'elements.json'), 'r') as f:
+    with open(os.path.join('test-datastore', uuid, 'elements.deflate'), 'rb') as f:
-        json.load(f)
+        import zlib
+        compressed_data = f.read()
+        decompressed_data = zlib.decompress(compressed_data)
+        # See if any error was thrown
+        json_data = json.loads(decompressed_data.decode('utf-8'))
 
     # Attempt to fetch it via the web hook that the browser would use
     res = client.get(url_for('static_content', group='visual_selector_data', filename=uuid))
-    json.loads(res.data)
+    decompressed_data = zlib.decompress(res.data)
+    json_data = json.loads(decompressed_data.decode('utf-8'))
+    
     assert res.mimetype == 'application/json'
     assert res.status_code == 200
 

changedetectionio/time_handler.py

@@ -0,0 +1,105 @@
+from datetime import timedelta, datetime
+from enum import IntEnum
+from zoneinfo import ZoneInfo
+
+
+class Weekday(IntEnum):
+    """Enumeration for days of the week."""
+    Monday = 0
+    Tuesday = 1
+    Wednesday = 2
+    Thursday = 3
+    Friday = 4
+    Saturday = 5
+    Sunday = 6
+
+
+def am_i_inside_time(
+        day_of_week: str,
+        time_str: str,
+        timezone_str: str,
+        duration: int = 15,
+) -> bool:
+    """
+    Determines if the current time falls within a specified time range.
+
+    Parameters:
+        day_of_week (str): The day of the week (e.g., 'Monday').
+        time_str (str): The start time in 'HH:MM' format.
+        timezone_str (str): The timezone identifier (e.g., 'Europe/Berlin').
+        duration (int, optional): The duration of the time range in minutes. Default is 15.
+
+    Returns:
+        bool: True if the current time is within the time range, False otherwise.
+    """
+    # Parse the target day of the week
+    try:
+        target_weekday = Weekday[day_of_week.capitalize()]
+    except KeyError:
+        raise ValueError(f"Invalid day_of_week: '{day_of_week}'. Must be a valid weekday name.")
+
+    # Parse the start time
+    try:
+        target_time = datetime.strptime(time_str, '%H:%M').time()
+    except ValueError:
+        raise ValueError(f"Invalid time_str: '{time_str}'. Must be in 'HH:MM' format.")
+
+    # Define the timezone
+    try:
+        tz = ZoneInfo(timezone_str)
+    except Exception:
+        raise ValueError(f"Invalid timezone_str: '{timezone_str}'. Must be a valid timezone identifier.")
+
+    # Get the current time in the specified timezone
+    now_tz = datetime.now(tz)
+
+    # Check if the current day matches the target day or overlaps due to duration
+    current_weekday = now_tz.weekday()
+    start_datetime_tz = datetime.combine(now_tz.date(), target_time, tzinfo=tz)
+
+    # Handle previous day's overlap
+    if target_weekday == (current_weekday - 1) % 7:
+        # Calculate start and end times for the overlap from the previous day
+        start_datetime_tz -= timedelta(days=1)
+        end_datetime_tz = start_datetime_tz + timedelta(minutes=duration)
+        if start_datetime_tz <= now_tz < end_datetime_tz:
+            return True
+
+    # Handle current day's range
+    if target_weekday == current_weekday:
+        end_datetime_tz = start_datetime_tz + timedelta(minutes=duration)
+        if start_datetime_tz <= now_tz < end_datetime_tz:
+            return True
+
+    # Handle next day's overlap
+    if target_weekday == (current_weekday + 1) % 7:
+        end_datetime_tz = start_datetime_tz + timedelta(minutes=duration)
+        if now_tz < start_datetime_tz and now_tz + timedelta(days=1) < end_datetime_tz:
+            return True
+
+    return False
+
+
+def is_within_schedule(time_schedule_limit, default_tz="UTC"):
+    if time_schedule_limit and time_schedule_limit.get('enabled'):
+        # Get the timezone the time schedule is in, so we know what day it is there
+        tz_name = time_schedule_limit.get('timezone')
+        if not tz_name:
+            tz_name = default_tz
+
+        now_day_name_in_tz = datetime.now(ZoneInfo(tz_name.strip())).strftime('%A')
+        selected_day_schedule = time_schedule_limit.get(now_day_name_in_tz.lower())
+        if not selected_day_schedule.get('enabled'):
+            return False
+
+        duration = selected_day_schedule.get('duration')
+        selected_day_run_duration_m = int(duration.get('hours')) * 60 + int(duration.get('minutes'))
+
+        is_valid = am_i_inside_time(day_of_week=now_day_name_in_tz,
+                                    time_str=selected_day_schedule['start_time'],
+                                    timezone_str=tz_name,
+                                    duration=selected_day_run_duration_m)
+
+        return is_valid
+
+    return False

changedetectionio/update_worker.py

@@ -28,6 +28,8 @@ class update_worker(threading.Thread):
 
     def queue_notification_for_watch(self, notification_q, n_object, watch):
         from changedetectionio import diff
+        from changedetectionio.notification import default_notification_format_for_watch
+
         dates = []
         trigger_text = ''
 
@@ -44,11 +46,21 @@ class update_worker(threading.Thread):
         else:
             snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
 
+        # If we ended up here with "System default"
+        if n_object.get('notification_format') == default_notification_format_for_watch:
+            n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')
+
+        html_colour_enable = False
         # HTML needs linebreak, but MarkDown and Text can use a linefeed
         if n_object.get('notification_format') == 'HTML':
             line_feed_sep = "<br>"
             # Snapshot will be plaintext on the disk, convert to some kind of HTML
             snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
+        elif n_object.get('notification_format') == 'HTML Color':
+            line_feed_sep = "<br>"
+            # Snapshot will be plaintext on the disk, convert to some kind of HTML
+            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
+            html_colour_enable = True
         else:
             line_feed_sep = "\n"
 
@@ -69,9 +81,9 @@ class update_worker(threading.Thread):
 
         n_object.update({
             'current_snapshot': snapshot_contents,
-            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep),
+            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
             'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
-            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep),
+            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
             'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
             'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
             'notification_timestamp': now,

docker-compose.yml

@@ -61,6 +61,12 @@ services:
   #
   #        If you want to watch local files file:///path/to/file.txt (careful! security implications!)
   #      - ALLOW_FILE_URI=False
+  #
+  #        For complete privacy if you don't want to use the 'check version' / telemetry service
+  #      - DISABLE_VERSION_CHECK=true
+  #
+  #        A valid timezone name to run as (for scheduling watch checking) see https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+  #      - TZ=America/Los_Angeles
   
       # Comment out ports: when using behind a reverse proxy , enable networks: etc.
       ports:

requirements.txt

@@ -1,7 +1,7 @@
 # Used by Pyppeteer
 pyee
 
-eventlet>=0.36.1 # fixes SSL error on Python 3.12
+eventlet>=0.38.0
 feedgen~=0.9
 flask-compress
 # 0.6.3 included compatibility fix for werkzeug 3.x (2.x had deprecation of url handlers)
@@ -38,9 +38,8 @@ dnspython==2.6.1 # related to eventlet fixes
 apprise==1.9.0
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
-# and 2.0.0 https://github.com/dgtlmoon/changedetection.io/issues/2241 not yet compatible
+# use any version other than 2.0.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
-# use v1.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
+paho-mqtt!=2.0.*
-paho-mqtt>=1.6.1,<2.0.0
 
 # Requires extra wheel for rPi
 cryptography~=42.0.8
@@ -96,4 +95,5 @@ babel
 # Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096
 greenlet >= 3.0.3
 
-
+# Scheduler - Windows seemed to miss a lot of default timezone info (even "UTC" !)
+tzdata