Watch history - Don't rescan whole history.txt when looking up a timestamp <->filepath

.github/workflows/test-stack-reusable-workflow.yml

@@ -69,7 +69,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -96,7 +96,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -135,7 +135,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -177,7 +177,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -217,7 +217,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -253,7 +253,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -282,7 +282,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -322,7 +322,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -353,7 +353,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -398,7 +398,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp

changedetectionio/PLUGIN_README.md

@@ -64,7 +64,7 @@ def count_words_in_history(watch):
             return 0
             
         latest_key = list(watch.history.keys())[-1]
-        latest_content = watch.get_history_snapshot(latest_key)
+        latest_content = watch.get_history_snapshot(timestamp=latest_key)
         return len(latest_content.split())
     except Exception as e:
         logger.error(f"Error counting words: {str(e)}")

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.50.37'
+__version__ = '0.50.39'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/api/Watch.py

@@ -175,7 +175,7 @@ class WatchSingleHistory(Resource):
                 response = make_response("No content found", 404)
                 response.mimetype = "text/plain"
         else:
-            content = watch.get_history_snapshot(timestamp)
+            content = watch.get_history_snapshot(timestamp=timestamp)
             response = make_response(content, 200)
             response.mimetype = "text/plain"
 

changedetectionio/api/api_schema.py

@@ -96,7 +96,10 @@ def build_watch_json_schema(d):
                                                            "enum": ["html_requests", "html_webdriver"]
                                                            })
 
-
+    schema['properties']['processor'] = {"anyOf": [
+        {"type": "string", "enum": ["restock_diff", "text_json_diff"]},
+        {"type": "null"}
+    ]}
 
     # All headers must be key/value type dict
     schema['properties']['headers'] = {

changedetectionio/blueprint/rss/blueprint.py

@@ -118,8 +118,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 fe.title(title=watch_label)
                 try:
 
-                    html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]),
-                                                 newest_version_file_contents=watch.get_history_snapshot(dates[-1]),
+                    html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(timestamp=dates[-2]),
+                                                 newest_version_file_contents=watch.get_history_snapshot(timestamp=dates[-1]),
                                                  include_equal=False,
                                                  line_feed_sep="<br>"
                                                  )

changedetectionio/blueprint/ui/notification.py

@@ -2,7 +2,7 @@ from flask import Blueprint, request, make_response
 import random
 from loguru import logger
 
-from changedetectionio.notification_service import NotificationContextData
+from changedetectionio.notification_service import NotificationContextData, set_basic_notification_vars
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.auth_decorator import login_optionally_required
 
@@ -95,7 +95,44 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 n_object['notification_body'] = "Test body"
 
             n_object['as_async'] = False
-            n_object.update(watch.extra_notification_token_values())
+
+            #  Same like in notification service, should be refactored
+            dates = []
+            trigger_text = ''
+            snapshot_contents = ''
+            if watch:
+                watch_history = watch.history
+                dates = list(watch_history.keys())
+                trigger_text = watch.get('trigger_text', [])
+                # Add text that was triggered
+                if len(dates):
+                    snapshot_contents = watch.get_history_snapshot(timestamp=dates[-1])
+                else:
+                    snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
+
+                if len(trigger_text):
+                    from . import html_tools
+                    triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
+                    if triggered_text:
+                        triggered_text = '\n'.join(triggered_text)
+
+            # Could be called as a 'test notification' with only 1 snapshot available
+            prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
+            current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
+
+
+
+            if len(dates) > 1:
+                prev_snapshot = watch.get_history_snapshot(timestamp=dates[-2])
+                current_snapshot = watch.get_history_snapshot(timestamp=dates[-1])
+
+            n_object.update(set_basic_notification_vars(snapshot_contents=snapshot_contents,
+                                                        current_snapshot=current_snapshot,
+                                                        prev_snapshot=prev_snapshot,
+                                                        watch=watch,
+                                                        triggered_text=trigger_text))
+
+
             sent_obj = process_notification(n_object, datastore)
 
         except Exception as e:

changedetectionio/blueprint/ui/views.py

@@ -47,7 +47,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
             try:
                 versions = list(watch.history.keys())
-                content = watch.get_history_snapshot(timestamp)
+                content = watch.get_history_snapshot(timestamp=timestamp)
 
                 triggered_line_numbers = html_tools.strip_ignore_text(content=content,
                                                                       wordlist=watch['trigger_text'],

changedetectionio/conditions/plugins/wordcount_plugin.py

@@ -14,7 +14,7 @@ def count_words_in_history(watch, incoming_text=None):
         elif watch.history.keys():
             # When called from UI extras to count latest snapshot
             latest_key = list(watch.history.keys())[-1]
-            latest_content = watch.get_history_snapshot(latest_key)
+            latest_content = watch.get_history_snapshot(timestamp=latest_key)
             return len(latest_content.split())
         return 0
     except Exception as e:

changedetectionio/flask_app.py

@@ -794,15 +794,19 @@ def ticker_thread_check_time_launch_checks():
 
             # @todo - Maybe make this a hook?
             # Time schedule limit - Decide between watch or global settings
+            scheduler_source = None
             if watch.get('time_between_check_use_default'):
                 time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {})
-                logger.trace(f"{uuid} Time scheduler - Using system/global settings")
+                scheduler_source = 'system/global settings'
+
             else:
                 time_schedule_limit = watch.get('time_schedule_limit')
-                logger.trace(f"{uuid} Time scheduler - Using watch settings (not global settings)")
+                scheduler_source = 'watch'
+
             tz_name = datastore.data['settings']['application'].get('scheduler_timezone_default', os.getenv('TZ', 'UTC').strip())
 
             if time_schedule_limit and time_schedule_limit.get('enabled'):
+                logger.trace(f"{uuid} Time scheduler - Using scheduler settings from {scheduler_source}")
                 try:
                     result = is_within_schedule(time_schedule_limit=time_schedule_limit,
                                                 default_tz=tz_name
@@ -814,6 +818,7 @@ def ticker_thread_check_time_launch_checks():
                     logger.error(
                         f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}")
                     return False
+
             # If they supplied an individual entry minutes to threshold.
             threshold = recheck_time_system_seconds if watch.get('time_between_check_use_default') else watch.threshold_seconds()
 

changedetectionio/model/Watch.py

@@ -276,9 +276,17 @@ class model(watch_base):
         # When the 'last viewed' timestamp is less than the oldest snapshot, return oldest
         return sorted_keys[-1]
 
-    def get_history_snapshot(self, timestamp):
+    def get_history_snapshot(self, timestamp=None, filepath=None):
+        """
+        Accepts either timestamp or filepath
+        :param timestamp:
+        :param filepath:
+        :return:
+        """
         import brotli
-        filepath = self.history[timestamp]
+
+        if not filepath:
+            filepath = self.history[timestamp]
 
         # See if a brotli versions exists and switch to that
         if not filepath.endswith('.br') and os.path.isfile(f"{filepath}.br"):
@@ -382,7 +390,7 @@ class model(watch_base):
         # Compare each lines (set) against each history text file (set) looking for something new..
         existing_history = set({})
         for k, v in self.history.items():
-            content = self.get_history_snapshot(k)
+            content = self.get_history_snapshot(filepath=v)
 
             if ignore_whitespace:
                 alist = set([line.translate(TRANSLATE_WHITESPACE_TABLE).lower() for line in content.splitlines()])
@@ -639,7 +647,7 @@ class model(watch_base):
         for k, fname in self.history.items():
             if os.path.isfile(fname):
                 if True:
-                    contents = self.get_history_snapshot(k)
+                    contents = self.get_history_snapshot(timestamp=k)
                     res = re.findall(regex, contents, re.MULTILINE)
                     if res:
                         if not csv_writer:
@@ -732,7 +740,7 @@ class model(watch_base):
             # If a previous attempt doesnt yet exist, just snarf the previous snapshot instead
             dates = list(self.history.keys())
             if len(dates):
-                return self.get_history_snapshot(dates[-1])
+                return self.get_history_snapshot(timestamp=dates[-1])
             else:
                 return ''
 

changedetectionio/notification_service.py

@@ -9,11 +9,8 @@ for both sync and async workers
 from loguru import logger
 import time
 
-from changedetectionio.model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
 from changedetectionio.notification import default_notification_format, valid_notification_formats
 
-# This gets modified on notification time (handler.py) depending on the required notification output
-CUSTOM_LINEBREAK_PLACEHOLDER='@BR@'
 
 
 # What is passed around as notification context, also used as the complete list of valid {{ tokens }}
@@ -23,10 +20,14 @@ class NotificationContextData(dict):
             'base_url': None,
             'current_snapshot': None,
             'diff': None,
+            'diff_clean': None,
             'diff_added': None,
+            'diff_added_clean': None,
             'diff_full': None,
+            'diff_full_clean': None,
             'diff_patch': None,
             'diff_removed': None,
+            'diff_removed_clean': None,
             'diff_url': None,
             'markup_text_links_to_html_links': False, # If automatic conversion of plaintext to HTML should happen
             'notification_timestamp': time.time(),
@@ -71,6 +72,38 @@ class NotificationContextData(dict):
 
         super().__setitem__(key, value)
 
+
+def set_basic_notification_vars(snapshot_contents, current_snapshot, prev_snapshot, watch, triggered_text):
+    now = time.time()
+    from changedetectionio import diff
+
+    n_object = {
+        'current_snapshot': snapshot_contents,
+        'diff': diff.render_diff(prev_snapshot, current_snapshot),
+        'diff_clean': diff.render_diff(prev_snapshot, current_snapshot, include_change_type_prefix=False),
+        'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False),
+        'diff_added_clean': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, include_change_type_prefix=False),
+        'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True),
+        'diff_full_clean': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, include_change_type_prefix=False),
+        'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, patch_format=True),
+        'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False),
+        'diff_removed_clean': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, include_change_type_prefix=False),
+        'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
+        'triggered_text': triggered_text,
+        'uuid': watch.get('uuid') if watch else None,
+        'watch_url': watch.get('url') if watch else None,
+        'watch_uuid': watch.get('uuid') if watch else None,
+        'watch_mime_type': watch.get('content-type')
+    }
+
+    # The \n's in the content from the above will get converted to <br> etc depending on the notification format
+
+    if watch:
+        n_object.update(watch.extra_notification_token_values())
+
+    logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time() - now:.3f}s")
+    return n_object
+
 class NotificationService:
     """
     Standalone notification service that handles all notification functionality
@@ -85,7 +118,6 @@ class NotificationService:
         """
         Queue a notification for a watch with full diff rendering and template variables
         """
-        from changedetectionio import diff
         from changedetectionio.notification import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
 
         if not isinstance(n_object, NotificationContextData):
@@ -94,8 +126,6 @@ class NotificationService:
         dates = []
         trigger_text = ''
 
-        now = time.time()
-
         if watch:
             watch_history = watch.history
             dates = list(watch_history.keys())
@@ -103,7 +133,7 @@ class NotificationService:
 
         # Add text that was triggered
         if len(dates):
-            snapshot_contents = watch.get_history_snapshot(dates[-1])
+            snapshot_contents = watch.get_history_snapshot(timestamp=dates[-1])
         else:
             snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
 
@@ -117,36 +147,23 @@ class NotificationService:
             from . import html_tools
             triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
             if triggered_text:
-                triggered_text = CUSTOM_LINEBREAK_PLACEHOLDER.join(triggered_text)
+                triggered_text = '\n'.join(triggered_text)
 
         # Could be called as a 'test notification' with only 1 snapshot available
         prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
         current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
 
         if len(dates) > 1:
-            prev_snapshot = watch.get_history_snapshot(dates[-2])
-            current_snapshot = watch.get_history_snapshot(dates[-1])
+            prev_snapshot = watch.get_history_snapshot(timestamp=dates[-2])
+            current_snapshot = watch.get_history_snapshot(timestamp=dates[-1])
 
-        n_object.update({
-            'current_snapshot': snapshot_contents,
-            'diff': diff.render_diff(prev_snapshot, current_snapshot),
-            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False),
-            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True),
-            'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, patch_format=True),
-            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False),
-            'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
-            'triggered_text': triggered_text,
-            'uuid': watch.get('uuid') if watch else None,
-            'watch_url': watch.get('url') if watch else None,
-            'watch_uuid': watch.get('uuid') if watch else None,
-            'watch_mime_type': watch.get('content-type')
-        })
-        # The \n's in the content from the above will get converted to <br> etc depending on the notification format
 
-        if watch:
-            n_object.update(watch.extra_notification_token_values())
+        n_object.update(set_basic_notification_vars(snapshot_contents=snapshot_contents,
+                                                    current_snapshot=current_snapshot,
+                                                    prev_snapshot=prev_snapshot,
+                                                    watch=watch,
+                                                    triggered_text=triggered_text))
 
-        logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
         logger.debug("Queued notification for sending")
         self.notification_q.put(n_object)
 

changedetectionio/static/styles/scss/styles.scss

@@ -329,12 +329,18 @@ a.pure-button-selected {
 .notifications-wrapper {
   padding-top: 0.5rem;
   #notification-test-log {
-    padding-top: 1rem;
+    margin-top: 1rem;
+    padding: 1rem;
     white-space: pre-wrap;
     word-break: break-word;
     overflow-wrap: break-word;
     max-width: 100%;
     box-sizing: border-box;
+    max-height: 12rem;
+    overflow-y: scroll;
+    border: 1px solid var(--color-border-notification);
+    border-radius: 5px;
+
   }
 }
 

changedetectionio/templates/_common_fields.html

@@ -87,19 +87,35 @@
 									<tr>
                                         <td><code>{{ '{{diff}}' }}</code></td>
                                         <td>The diff output - only changes, additions, and removals</td>
+                                    </tr>
+									<tr>
+                                        <td><code>{{ '{{diff_clean}}' }}</code></td>
+                                        <td>The diff output - only changes, additions, and removals &dash; <i>Without (added) prefix or colors</i></td>
                                     </tr>
 									<tr>
                                         <td><code>{{ '{{diff_added}}' }}</code></td>
                                         <td>The diff output - only changes and additions</td>
+                                    </tr>
+									<tr>
+                                        <td><code>{{ '{{diff_added_clean}}' }}</code></td>
+                                        <td>The diff output - only changes and additions &dash; <i>Without (added) prefix or colors</i></td>
                                     </tr>
 									<tr>
                                         <td><code>{{ '{{diff_removed}}' }}</code></td>
                                         <td>The diff output - only changes and removals</td>
+                                    </tr>
+									<tr>
+                                        <td><code>{{ '{{diff_removed_clean}}' }}</code></td>
+                                        <td>The diff output - only changes and removals &dash; <i>Without (added) prefix or colors</i></td>
                                     </tr>
                                     <tr>
                                         <td><code>{{ '{{diff_full}}' }}</code></td>
                                         <td>The diff output - full difference output</td>
                                     </tr>
+                                    <tr>
+                                        <td><code>{{ '{{diff_full_clean}}' }}</code></td>
+                                        <td>The diff output - full difference output &dash; <i>Without (added) prefix or colors</i></td>
+                                    </tr>
                                     <tr>
                                         <td><code>{{ '{{diff_patch}}' }}</code></td>
                                         <td>The diff output - patch in unified format</td>

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -158,6 +158,7 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
     assert ADDED_PLACEMARKER_OPEN not in subject
     assert 'diff added didnt split' not in subject
     assert '(changed) Which is across' in subject
+    assert 'PLACEMARKER' not in subject
 
     # The email should be plain text only (not multipart)
     assert not msg.is_multipart()
@@ -226,6 +227,7 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
     assert ADDED_PLACEMARKER_OPEN not in subject
     assert 'diff added didnt split' not in subject
     assert '(changed) Which is across' in subject
+    assert 'PLACEMARKER' not in subject
     assert 'head title' in subject
     assert "span" not in subject
     assert 'background-color' not in subject
@@ -583,6 +585,7 @@ def test_check_plaintext_document_html_notifications(client, live_server, measur
     # Should be the HTML, but not HTML Color
     assert 'background-color' not in html_content
     assert '<br>(added) And let&#39;s talk about &lt;title&gt; tags<br>' in html_content
+    assert 'PLACEMARKER' not in html_content
     assert '&lt;br' not in html_content
     assert '<pre role="article"' in html_content # Should have got wrapped nicely in email_helpers.py
 
@@ -713,6 +716,7 @@ def test_check_html_document_plaintext_notification(client, live_server, measure
 
     assert '<tag>' in body # Should have got converted from original HTML to plaintext
     assert '(changed) some stuff\r\n' in body
+    assert 'PLACEMARKER' not in body
     assert '(into) sxome stuff\r\n' in body
     assert '(added) lets slip this in\r\n' in body
     assert '(added) and this in\r\n' in body

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -353,7 +353,7 @@ def check_json_ext_filter(json_filter, client, live_server, datastore_path):
 
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
     dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(dates[0])
+    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
 
     assert snapshot_contents[0] == '['
 
@@ -439,7 +439,7 @@ def test_correct_header_detect(client, live_server, measure_memory_usage, datast
 
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
     dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(dates[0])
+    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
 
     assert b'"hello": 123,' in res.data # properly html escaped in the front end
 

changedetectionio/tests/test_notification.py

@@ -404,15 +404,15 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
 
 
 #2510
+#@todo run it again as text, html, htmlcolor
 def test_global_send_test_notification(client, live_server, measure_memory_usage, datastore_path):
 
-    
     set_original_response(datastore_path=datastore_path)
     if os.path.isfile(os.path.join(datastore_path, "notification.txt")):
         os.unlink(os.path.join(datastore_path, "notification.txt")) \
 
     # 1995 UTF-8 content should be encoded
-    test_body = 'change detection is cool 网站监测 内容更新了'
+    test_body = 'change detection is cool 网站监测 内容更新了 - {{diff_full}}'
 
     # otherwise other settings would have already existed from previous tests in this file
     res = client.post(
@@ -452,7 +452,14 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
 
     with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
         x = f.read()
-        assert test_body in x
+        assert 'change detection is cool 网站监测 内容更新了' in x
+        if 'html' in default_notification_format:
+            # this should come from default text when in global/system mode here changedetectionio/notification_service.py
+            assert 'title="Changed into">Example text:' in x
+        else:
+            assert 'title="Changed into">Example text:' not in x
+            assert 'span' not in x
+            assert 'Example text:' in x
 
     os.unlink(os.path.join(datastore_path, "notification.txt"))
 
@@ -509,6 +516,47 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert b"Error: You must have atleast one watch configured for 'test notification' to work" in res.data
 
 
+#2510
+def test_single_send_test_notification_on_watch(client, live_server, measure_memory_usage, datastore_path):
+
+    set_original_response(datastore_path=datastore_path)
+    if os.path.isfile(os.path.join(datastore_path, "notification.txt")):
+        os.unlink(os.path.join(datastore_path, "notification.txt")) \
+
+
+    test_url = url_for('test_endpoint', _external=True)
+    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
+    # 1995 UTF-8 content should be encoded
+    test_body = 'change detection is cool 网站监测 内容更新了 - {{diff_full}}'
+    ######### Test global/system settings
+    res = client.post(
+        url_for("ui.ui_notification.ajax_callback_send_notification_test")+f"/{uuid}",
+        data={"notification_urls": test_notification_url,
+              "notification_body": test_body,
+              "notification_format": default_notification_format,
+              "notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
+              },
+        follow_redirects=True
+    )
+
+    assert res.status_code != 400
+    assert res.status_code != 500
+
+    with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
+        x = f.read()
+        assert 'change detection is cool 网站监测 内容更新了' in x
+        if 'html' in default_notification_format:
+            # this should come from default text when in global/system mode here changedetectionio/notification_service.py
+            assert 'title="Changed into">Example text:' in x
+        else:
+            assert 'title="Changed into">Example text:' not in x
+            assert 'span' not in x
+            assert 'Example text:' in x
+
+    os.unlink(os.path.join(datastore_path, "notification.txt"))
 
 def _test_color_notifications(client, notification_body_token, datastore_path):
 

changedetectionio/tests/test_pdf.py

@@ -22,7 +22,7 @@ def test_fetch_pdf(client, live_server, measure_memory_usage, datastore_path):
 
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
     dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(dates[0])
+    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
 
     # PDF header should not be there (it was converted to text)
     assert 'PDF' not in snapshot_contents
@@ -75,7 +75,7 @@ def test_fetch_pdf(client, live_server, measure_memory_usage, datastore_path):
 
     dates = list(watch.history.keys())
     # new snapshot was also OK, no HTML
-    snapshot_contents = watch.get_history_snapshot(dates[1])
+    snapshot_contents = watch.get_history_snapshot(timestamp=dates[1])
     assert 'html' not in snapshot_contents.lower()
     assert f'Original file size - {os.path.getsize(os.path.join(datastore_path, "endpoint-test.pdf"))}' in snapshot_contents
     assert f'here is a change' in snapshot_contents

changedetectionio/tests/test_rss_reader_mode.py

@@ -65,7 +65,7 @@ def test_rss_reader_mode(client, live_server, measure_memory_usage, datastore_pa
 
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
     dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(dates[0])
+    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
     assert 'Wet noodles escape' in snapshot_contents
     assert '<br>' not in snapshot_contents
     assert '&lt;' not in snapshot_contents
@@ -91,7 +91,7 @@ def test_rss_reader_mode_with_css_filters(client, live_server, measure_memory_us
 
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
     dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(dates[0])
+    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
     assert 'Wet noodles escape' not in snapshot_contents
     assert '<br>' not in snapshot_contents
     assert '&lt;' not in snapshot_contents

changedetectionio/tests/unit/test_conditions.py

@@ -55,8 +55,8 @@ class TestTriggerConditions(unittest.TestCase):
         self.assertEqual(len(history), 2)
 
         # Retrieve and check snapshots
-        #snapshot1 = watch.get_history_snapshot(str(timestamp1))
-        #snapshot2 = watch.get_history_snapshot(str(timestamp2))
+        #snapshot1 = watch.get_history_snapshot(timestamp=str(timestamp1))
+        #snapshot2 = watch.get_history_snapshot(timestamp=str(timestamp2))
 
         self.store.data['watching'][self.watch_uuid].update(
             {

changedetectionio/time_handler.py

@@ -14,7 +14,6 @@ class Weekday(IntEnum):
     Saturday = 5
     Sunday = 6
 
-@lru_cache(maxsize=100)
 def am_i_inside_time(
         day_of_week: str,
         time_str: str,

docs/api-spec.yaml

@@ -28,7 +28,7 @@ info:
     
     For example: `x-api-key: YOUR_API_KEY`
     
-  version: 0.1.2
+  version: 0.1.3
   contact:
     name: ChangeDetection.io
     url: https://github.com/dgtlmoon/changedetection.io
@@ -65,13 +65,17 @@ tags:
       
   - name: Watch History
     description: |
-      Access historical snapshots and change data for your watches. View the complete timeline of detected changes 
-      and retrieve specific versions of monitored content for comparison and analysis.
+      Get a list of timestamps of all changes detected for a watch.
       
   - name: Snapshots
     description: |
-      Retrieve individual snapshots of monitored content. Access both the processed change detection data and 
-      the raw HTML content that was captured during monitoring checks.
+      Retrieve individual text snapshot of monitored content according to the `timestamp`. The text snapshot is the HTML
+      to Text at page check time. 
+      
+      Set the query argument `html` to any value to retrieve the last HTML fetched, the system only keeps the last two 
+      (2) HTML files fetched.
+      
+      Use the Watch History API endpoint to get a list of timestamps to pass to this query.
       
   - name: Favicon
     description: |
@@ -228,6 +232,11 @@ components:
                 maxLength: 5000
             required: [operation, selector, optional_value]
           description: Browser automation steps
+        processor:
+          type: string
+          enum: [restock_diff, text_json_diff]
+          default: text_json_diff
+          description: Optional processor mode to use for change detection. Defaults to `text_json_diff` if not specified.
 
     Watch:
       allOf:
@@ -428,7 +437,15 @@ paths:
       operationId: createWatch
       tags: [Watch Management]
       summary: Create a new watch
-      description: Create a single web page change monitor (watch). Requires at least 'url' to be set.
+      description: |
+        Create a single web page change monitor (watch). Requires at least `url` to be set.
+
+        Every watch can be configured with:
+        - **Processor mode**: `processor` field (`restock_diff` or `text_json_diff` - default)
+        - **Notification settings**: `notification_urls` (array), `notification_title`, `notification_body`, `notification_format`, `notification_muted`
+        - **Tags/Groups**: `tag` (UUID string) or `tags` (array of UUIDs)
+        - **Check settings**: `time_between_check`, `paused`, `method`, `fetch_backend`
+        - **Advanced options**: `headers`, `body`, `proxy`, `browser_steps`, and more
       x-code-samples:
         - lang: 'curl'
           source: |
@@ -446,7 +463,7 @@ paths:
           source: |
             import requests
             import json
-            
+
             headers = {
                 'x-api-key': 'YOUR_API_KEY',
                 'Content-Type': 'application/json'
@@ -458,7 +475,7 @@ paths:
                     'hours': 1
                 }
             }
-            response = requests.post('http://localhost:5000/api/v1/watch', 
+            response = requests.post('http://localhost:5000/api/v1/watch',
                                    headers=headers, json=data)
             print(response.text)
       requestBody:
@@ -648,7 +665,9 @@ paths:
       operationId: getWatchHistory
       tags: [Watch History]
       summary: Get watch history
-      description: Get a list of all historical snapshots available for a web page change monitor (watch)
+      description: | 
+        Get a list of all historical snapshots available for a web page change monitor (watch), use the key `timestamp`
+        as the query argument for fetching a single watch history snapshot.
       x-code-samples:
         - lang: 'curl'
           source: |
@@ -688,7 +707,9 @@ paths:
       operationId: getWatchSnapshot
       tags: [Snapshots]
       summary: Get single snapshot
-      description: Get single snapshot from web page change monitor (watch). Use 'latest' for the most recent snapshot.
+      description: | 
+        Get single snapshot from web page change monitor (watch). Use 'latest' for the most recent snapshot.
+        Use the Watch History API to get a list of timestamps to pass.
       x-code-samples:
         - lang: 'curl'
           source: |

requirements.txt

@@ -12,7 +12,7 @@ janus # Thread-safe async/sync queue bridge
 flask_wtf~=1.2
 flask~=3.1
 flask-socketio~=5.5.1
-python-socketio~=5.14.2
+python-socketio~=5.14.3
 python-engineio~=4.12.3
 inscriptis~=2.2
 pytz
@@ -31,7 +31,7 @@ requests-file
 chardet>2.3.0
 
 wtforms~=3.2
-jsonpath-ng~=1.5.3
+jsonpath-ng~=1.7.0
 
 # dnspython - Used by paho-mqtt for MQTT broker resolution  
 # Version pin removed since eventlet (which required the specific 2.6.1 pin) has been eliminated
@@ -87,7 +87,7 @@ pyppeteerstealth>=0.0.4
 
 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup
 pytest ~=7.2
-pytest-flask ~=1.2
+pytest-flask ~=1.3
 pytest-mock ~=3.15
 
 # Anything 4.0 and up but not 5.0