Watch history - Don't rescan whole history.txt when looking up a timestamp <->filepath

.github/workflows/test-stack-reusable-workflow.yml

@@ -69,7 +69,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -96,7 +96,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -135,7 +135,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -177,7 +177,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -217,7 +217,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -253,7 +253,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -282,7 +282,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -322,7 +322,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -353,7 +353,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -398,7 +398,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp

changedetectionio/PLUGIN_README.md

@@ -64,7 +64,7 @@ def count_words_in_history(watch):
             return 0
             
         latest_key = list(watch.history.keys())[-1]
-        latest_content = watch.get_history_snapshot(latest_key)
+        latest_content = watch.get_history_snapshot(timestamp=latest_key)
         return len(latest_content.split())
     except Exception as e:
         logger.error(f"Error counting words: {str(e)}")

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.50.38'
+__version__ = '0.50.39'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/api/Watch.py

@@ -175,7 +175,7 @@ class WatchSingleHistory(Resource):
                 response = make_response("No content found", 404)
                 response.mimetype = "text/plain"
         else:
-            content = watch.get_history_snapshot(timestamp)
+            content = watch.get_history_snapshot(timestamp=timestamp)
             response = make_response(content, 200)
             response.mimetype = "text/plain"
 

changedetectionio/blueprint/rss/blueprint.py

@@ -118,8 +118,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 fe.title(title=watch_label)
                 try:
 
-                    html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]),
+                    html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(timestamp=dates[-2]),
-                                                 newest_version_file_contents=watch.get_history_snapshot(dates[-1]),
+                                                 newest_version_file_contents=watch.get_history_snapshot(timestamp=dates[-1]),
                                                  include_equal=False,
                                                  line_feed_sep="<br>"
                                                  )

changedetectionio/blueprint/ui/notification.py

@@ -106,7 +106,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 trigger_text = watch.get('trigger_text', [])
                 # Add text that was triggered
                 if len(dates):
-                    snapshot_contents = watch.get_history_snapshot(dates[-1])
+                    snapshot_contents = watch.get_history_snapshot(timestamp=dates[-1])
                 else:
                     snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
 
@@ -123,8 +123,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
 
             if len(dates) > 1:
-                prev_snapshot = watch.get_history_snapshot(dates[-2])
+                prev_snapshot = watch.get_history_snapshot(timestamp=dates[-2])
-                current_snapshot = watch.get_history_snapshot(dates[-1])
+                current_snapshot = watch.get_history_snapshot(timestamp=dates[-1])
 
             n_object.update(set_basic_notification_vars(snapshot_contents=snapshot_contents,
                                                         current_snapshot=current_snapshot,

changedetectionio/blueprint/ui/views.py

@@ -47,7 +47,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
             try:
                 versions = list(watch.history.keys())
-                content = watch.get_history_snapshot(timestamp)
+                content = watch.get_history_snapshot(timestamp=timestamp)
 
                 triggered_line_numbers = html_tools.strip_ignore_text(content=content,
                                                                       wordlist=watch['trigger_text'],

changedetectionio/conditions/plugins/wordcount_plugin.py

@@ -14,7 +14,7 @@ def count_words_in_history(watch, incoming_text=None):
         elif watch.history.keys():
             # When called from UI extras to count latest snapshot
             latest_key = list(watch.history.keys())[-1]
-            latest_content = watch.get_history_snapshot(latest_key)
+            latest_content = watch.get_history_snapshot(timestamp=latest_key)
             return len(latest_content.split())
         return 0
     except Exception as e:

changedetectionio/flask_app.py

@@ -794,15 +794,19 @@ def ticker_thread_check_time_launch_checks():
 
             # @todo - Maybe make this a hook?
             # Time schedule limit - Decide between watch or global settings
+            scheduler_source = None
             if watch.get('time_between_check_use_default'):
                 time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {})
-                logger.trace(f"{uuid} Time scheduler - Using system/global settings")
+                scheduler_source = 'system/global settings'
+
             else:
                 time_schedule_limit = watch.get('time_schedule_limit')
-                logger.trace(f"{uuid} Time scheduler - Using watch settings (not global settings)")
+                scheduler_source = 'watch'
+
             tz_name = datastore.data['settings']['application'].get('scheduler_timezone_default', os.getenv('TZ', 'UTC').strip())
 
             if time_schedule_limit and time_schedule_limit.get('enabled'):
+                logger.trace(f"{uuid} Time scheduler - Using scheduler settings from {scheduler_source}")
                 try:
                     result = is_within_schedule(time_schedule_limit=time_schedule_limit,
                                                 default_tz=tz_name
@@ -814,6 +818,7 @@ def ticker_thread_check_time_launch_checks():
                     logger.error(
                         f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}")
                     return False
+
             # If they supplied an individual entry minutes to threshold.
             threshold = recheck_time_system_seconds if watch.get('time_between_check_use_default') else watch.threshold_seconds()
 

changedetectionio/model/Watch.py

@@ -276,9 +276,17 @@ class model(watch_base):
         # When the 'last viewed' timestamp is less than the oldest snapshot, return oldest
         return sorted_keys[-1]
 
-    def get_history_snapshot(self, timestamp):
+    def get_history_snapshot(self, timestamp=None, filepath=None):
+        """
+        Accepts either timestamp or filepath
+        :param timestamp:
+        :param filepath:
+        :return:
+        """
         import brotli
-        filepath = self.history[timestamp]
+
+        if not filepath:
+            filepath = self.history[timestamp]
 
         # See if a brotli versions exists and switch to that
         if not filepath.endswith('.br') and os.path.isfile(f"{filepath}.br"):
@@ -382,7 +390,7 @@ class model(watch_base):
         # Compare each lines (set) against each history text file (set) looking for something new..
         existing_history = set({})
         for k, v in self.history.items():
-            content = self.get_history_snapshot(k)
+            content = self.get_history_snapshot(filepath=v)
 
             if ignore_whitespace:
                 alist = set([line.translate(TRANSLATE_WHITESPACE_TABLE).lower() for line in content.splitlines()])
@@ -639,7 +647,7 @@ class model(watch_base):
         for k, fname in self.history.items():
             if os.path.isfile(fname):
                 if True:
-                    contents = self.get_history_snapshot(k)
+                    contents = self.get_history_snapshot(timestamp=k)
                     res = re.findall(regex, contents, re.MULTILINE)
                     if res:
                         if not csv_writer:
@@ -732,7 +740,7 @@ class model(watch_base):
             # If a previous attempt doesnt yet exist, just snarf the previous snapshot instead
             dates = list(self.history.keys())
             if len(dates):
-                return self.get_history_snapshot(dates[-1])
+                return self.get_history_snapshot(timestamp=dates[-1])
             else:
                 return ''
 

changedetectionio/notification_service.py

@@ -133,7 +133,7 @@ class NotificationService:
 
         # Add text that was triggered
         if len(dates):
-            snapshot_contents = watch.get_history_snapshot(dates[-1])
+            snapshot_contents = watch.get_history_snapshot(timestamp=dates[-1])
         else:
             snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
 
@@ -154,8 +154,8 @@ class NotificationService:
         current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
 
         if len(dates) > 1:
-            prev_snapshot = watch.get_history_snapshot(dates[-2])
+            prev_snapshot = watch.get_history_snapshot(timestamp=dates[-2])
-            current_snapshot = watch.get_history_snapshot(dates[-1])
+            current_snapshot = watch.get_history_snapshot(timestamp=dates[-1])
 
 
         n_object.update(set_basic_notification_vars(snapshot_contents=snapshot_contents,

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -158,6 +158,7 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
     assert ADDED_PLACEMARKER_OPEN not in subject
     assert 'diff added didnt split' not in subject
     assert '(changed) Which is across' in subject
+    assert 'PLACEMARKER' not in subject
 
     # The email should be plain text only (not multipart)
     assert not msg.is_multipart()
@@ -226,6 +227,7 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
     assert ADDED_PLACEMARKER_OPEN not in subject
     assert 'diff added didnt split' not in subject
     assert '(changed) Which is across' in subject
+    assert 'PLACEMARKER' not in subject
     assert 'head title' in subject
     assert "span" not in subject
     assert 'background-color' not in subject
@@ -583,6 +585,7 @@ def test_check_plaintext_document_html_notifications(client, live_server, measur
     # Should be the HTML, but not HTML Color
     assert 'background-color' not in html_content
     assert '<br>(added) And let&#39;s talk about &lt;title&gt; tags<br>' in html_content
+    assert 'PLACEMARKER' not in html_content
     assert '&lt;br' not in html_content
     assert '<pre role="article"' in html_content # Should have got wrapped nicely in email_helpers.py
 
@@ -713,6 +716,7 @@ def test_check_html_document_plaintext_notification(client, live_server, measure
 
     assert '<tag>' in body # Should have got converted from original HTML to plaintext
     assert '(changed) some stuff\r\n' in body
+    assert 'PLACEMARKER' not in body
     assert '(into) sxome stuff\r\n' in body
     assert '(added) lets slip this in\r\n' in body
     assert '(added) and this in\r\n' in body

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -353,7 +353,7 @@ def check_json_ext_filter(json_filter, client, live_server, datastore_path):
 
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
     dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(dates[0])
+    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
 
     assert snapshot_contents[0] == '['
 
@@ -439,7 +439,7 @@ def test_correct_header_detect(client, live_server, measure_memory_usage, datast
 
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
     dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(dates[0])
+    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
 
     assert b'"hello": 123,' in res.data # properly html escaped in the front end
 

changedetectionio/tests/test_pdf.py

@@ -22,7 +22,7 @@ def test_fetch_pdf(client, live_server, measure_memory_usage, datastore_path):
 
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
     dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(dates[0])
+    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
 
     # PDF header should not be there (it was converted to text)
     assert 'PDF' not in snapshot_contents
@@ -75,7 +75,7 @@ def test_fetch_pdf(client, live_server, measure_memory_usage, datastore_path):
 
     dates = list(watch.history.keys())
     # new snapshot was also OK, no HTML
-    snapshot_contents = watch.get_history_snapshot(dates[1])
+    snapshot_contents = watch.get_history_snapshot(timestamp=dates[1])
     assert 'html' not in snapshot_contents.lower()
     assert f'Original file size - {os.path.getsize(os.path.join(datastore_path, "endpoint-test.pdf"))}' in snapshot_contents
     assert f'here is a change' in snapshot_contents

changedetectionio/tests/test_rss_reader_mode.py

@@ -65,7 +65,7 @@ def test_rss_reader_mode(client, live_server, measure_memory_usage, datastore_pa
 
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
     dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(dates[0])
+    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
     assert 'Wet noodles escape' in snapshot_contents
     assert '<br>' not in snapshot_contents
     assert '&lt;' not in snapshot_contents
@@ -91,7 +91,7 @@ def test_rss_reader_mode_with_css_filters(client, live_server, measure_memory_us
 
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
     dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(dates[0])
+    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
     assert 'Wet noodles escape' not in snapshot_contents
     assert '<br>' not in snapshot_contents
     assert '&lt;' not in snapshot_contents

changedetectionio/tests/unit/test_conditions.py

@@ -55,8 +55,8 @@ class TestTriggerConditions(unittest.TestCase):
         self.assertEqual(len(history), 2)
 
         # Retrieve and check snapshots
-        #snapshot1 = watch.get_history_snapshot(str(timestamp1))
+        #snapshot1 = watch.get_history_snapshot(timestamp=str(timestamp1))
-        #snapshot2 = watch.get_history_snapshot(str(timestamp2))
+        #snapshot2 = watch.get_history_snapshot(timestamp=str(timestamp2))
 
         self.store.data['watching'][self.watch_uuid].update(
             {

changedetectionio/time_handler.py

@@ -14,7 +14,6 @@ class Weekday(IntEnum):
     Saturday = 5
     Sunday = 6
 
-@lru_cache(maxsize=100)
 def am_i_inside_time(
         day_of_week: str,
         time_str: str,

docs/api-spec.yaml

@@ -28,7 +28,7 @@ info:
     
     For example: `x-api-key: YOUR_API_KEY`
     
-  version: 0.1.2
+  version: 0.1.3
   contact:
     name: ChangeDetection.io
     url: https://github.com/dgtlmoon/changedetection.io
@@ -65,13 +65,17 @@ tags:
       
   - name: Watch History
     description: |
-      Access historical snapshots and change data for your watches. View the complete timeline of detected changes 
+      Get a list of timestamps of all changes detected for a watch.
-      and retrieve specific versions of monitored content for comparison and analysis.
       
   - name: Snapshots
     description: |
-      Retrieve individual snapshots of monitored content. Access both the processed change detection data and 
+      Retrieve individual text snapshot of monitored content according to the `timestamp`. The text snapshot is the HTML
-      the raw HTML content that was captured during monitoring checks.
+      to Text at page check time. 
+      
+      Set the query argument `html` to any value to retrieve the last HTML fetched, the system only keeps the last two 
+      (2) HTML files fetched.
+      
+      Use the Watch History API endpoint to get a list of timestamps to pass to this query.
       
   - name: Favicon
     description: |
@@ -433,7 +437,15 @@ paths:
       operationId: createWatch
       tags: [Watch Management]
       summary: Create a new watch
-      description: Create a single web page change monitor (watch). Requires at least 'url' to be set, Optionally use `"processor"` field to set the `restock_diff` mode or `text_json_diff` (default)
+      description: |
+        Create a single web page change monitor (watch). Requires at least `url` to be set.
+
+        Every watch can be configured with:
+        - **Processor mode**: `processor` field (`restock_diff` or `text_json_diff` - default)
+        - **Notification settings**: `notification_urls` (array), `notification_title`, `notification_body`, `notification_format`, `notification_muted`
+        - **Tags/Groups**: `tag` (UUID string) or `tags` (array of UUIDs)
+        - **Check settings**: `time_between_check`, `paused`, `method`, `fetch_backend`
+        - **Advanced options**: `headers`, `body`, `proxy`, `browser_steps`, and more
       x-code-samples:
         - lang: 'curl'
           source: |
@@ -653,7 +665,9 @@ paths:
       operationId: getWatchHistory
       tags: [Watch History]
       summary: Get watch history
-      description: Get a list of all historical snapshots available for a web page change monitor (watch)
+      description: | 
+        Get a list of all historical snapshots available for a web page change monitor (watch), use the key `timestamp`
+        as the query argument for fetching a single watch history snapshot.
       x-code-samples:
         - lang: 'curl'
           source: |
@@ -693,7 +707,9 @@ paths:
       operationId: getWatchSnapshot
       tags: [Snapshots]
       summary: Get single snapshot
-      description: Get single snapshot from web page change monitor (watch). Use 'latest' for the most recent snapshot.
+      description: | 
+        Get single snapshot from web page change monitor (watch). Use 'latest' for the most recent snapshot.
+        Use the Watch History API to get a list of timestamps to pass.
       x-code-samples:
         - lang: 'curl'
           source: |

requirements.txt

@@ -12,7 +12,7 @@ janus # Thread-safe async/sync queue bridge
 flask_wtf~=1.2
 flask~=3.1
 flask-socketio~=5.5.1
-python-socketio~=5.14.2
+python-socketio~=5.14.3
 python-engineio~=4.12.3
 inscriptis~=2.2
 pytz
@@ -31,7 +31,7 @@ requests-file
 chardet>2.3.0
 
 wtforms~=3.2
-jsonpath-ng~=1.5.3
+jsonpath-ng~=1.7.0
 
 # dnspython - Used by paho-mqtt for MQTT broker resolution  
 # Version pin removed since eventlet (which required the specific 2.6.1 pin) has been eliminated
@@ -87,7 +87,7 @@ pyppeteerstealth>=0.0.4
 
 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup
 pytest ~=7.2
-pytest-flask ~=1.2
+pytest-flask ~=1.3
 pytest-mock ~=3.15
 
 # Anything 4.0 and up but not 5.0