tweaks

* improving unique key fix

* also bump timestamp along 1 sec

.github/workflows/containers.yml

@@ -40,10 +40,10 @@ jobs:
     if: ${{ github.event.workflow_run.conclusion == 'success' }} || ${{ github.event.release.tag_name }} != ''
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.12
+      - name: Set up Python 3.11
         uses: actions/setup-python@v5
         with:
-          python-version: 3.12
+          python-version: 3.11
 
       - name: Install dependencies
         run: |

.github/workflows/pypi-release.yml

@@ -11,7 +11,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.12"
+        python-version: "3.11"
     - name: Install pypa/build
       run: >-
         python3 -m
@@ -38,10 +38,10 @@ jobs:
       with:
         name: python-package-distributions
         path: dist/
-    - name: Set up Python 3.12
+    - name: Set up Python 3.11
       uses: actions/setup-python@v5
       with:
-        python-version: '3.12'
+        python-version: '3.11'
     - name: Test that the basic pip built package runs without error
       run: |
         set -ex

.github/workflows/test-container-build.yml

@@ -27,10 +27,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
         - uses: actions/checkout@v4
-        - name: Set up Python 3.12
+        - name: Set up Python 3.11
           uses: actions/setup-python@v5
           with:
-            python-version: 3.12
+            python-version: 3.11
 
         # Just test that the build works, some libraries won't compile on ARM/rPi etc
         - name: Set up QEMU

.github/workflows/test-only.yml

@@ -10,10 +10,10 @@ jobs:
       - uses: actions/checkout@v4
 
       # Mainly just for link/flake8
-      - name: Set up Python 3.12
+      - name: Set up Python 3.11
         uses: actions/setup-python@v5
         with:
-          python-version: '3.12'
+          python-version: '3.11'
 
       - name: Lint with flake8
         run: |

Dockerfile

@@ -2,7 +2,7 @@
 
 # @NOTE! I would love to move to 3.11 but it breaks the async handler in changedetectionio/content_fetchers/puppeteer.py
 #        If you know how to fix it, please do! and test it for both 3.10 and 3.11
-FROM python:3.12-slim-bookworm as builder
+FROM python:3.10-slim-bookworm as builder
 
 # See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -32,7 +32,7 @@ RUN pip install --target=/dependencies playwright~=1.41.2 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.12-slim-bookworm
+FROM python:3.10-slim-bookworm
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libxslt1.1 \

changedetectionio/flask_app.py

@@ -45,7 +45,6 @@ running_update_threads = []
 ticker_thread = None
 
 extra_stylesheets = []
-
 update_q = queue.PriorityQueue()
 notification_q = queue.Queue()
 
@@ -1538,6 +1537,11 @@ def changedetection_app(config=None, datastore_o=None):
         # paste in etc
         return redirect(url_for('index'))
 
+    @app.route("/queue_size", methods=['GET'])
+    @login_optionally_required
+    def get_queue_size():
+        return update_q.qsize()
+
     @app.route("/highlight_submit_ignore_url", methods=['POST'])
     @login_optionally_required
     def highlight_submit_ignore_url():

changedetectionio/model/Watch.py

@@ -1,5 +1,6 @@
 from changedetectionio.strtobool import strtobool
 from changedetectionio.safe_jinja import render as jinja_render
+
 import os
 import re
 import time
@@ -216,6 +217,8 @@ class model(dict):
         fname = os.path.join(self.watch_data_dir, "history.txt")
         if os.path.isfile(fname):
             logger.debug(f"Reading watch history index for {self.get('uuid')}")
+
+
             with open(fname, "r") as f:
                 for i in f.readlines():
                     if ',' in i:
@@ -332,7 +335,9 @@ class model(dict):
         # Small hack so that we sleep just enough to allow 1 second  between history snapshots
         # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
         if self.__newest_history_key and int(timestamp) == int(self.__newest_history_key):
-            time.sleep(timestamp - self.__newest_history_key)
+            logger.warning(f"Timestamp {timestamp} already exists, waiting 1 seconds so we have a unique key in history.txt")
+            timestamp = str(int(timestamp) + 1)
+            time.sleep(1)
 
         threshold = int(os.getenv('SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD', 1024))
         skip_brotli = strtobool(os.getenv('DISABLE_BROTLI_TEXT_SNAPSHOT', 'False'))

changedetectionio/store.py

@@ -178,7 +178,7 @@ class ChangeDetectionStore:
     @property
     def has_unviewed(self):
         for uuid, watch in self.__data['watching'].items():
-            if watch.viewed == False:
+            if watch.history_n >= 2 and watch.viewed == False:
                 return True
         return False
 

changedetectionio/tests/restock/test_restock.py

@@ -53,7 +53,6 @@ def test_restock_detection(client, live_server):
     set_original_response()
     #assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
 
-    time.sleep(1)
     live_server_setup(live_server)
     #####################
     notification_url = url_for('test_notification_endpoint', _external=True).replace('http://localhost', 'http://changedet').replace('http', 'json')

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -71,6 +71,8 @@ def test_check_notification_email_formats_default_HTML(client, live_server):
 
     wait_for_all_checks(client)
     set_longer_modified_response()
+    time.sleep(2)
+
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -135,6 +137,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
     wait_for_all_checks(client)
     set_longer_modified_response()
+    time.sleep(2)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -40,8 +40,6 @@ def test_setup(client, live_server):
 
 def test_check_removed_line_contains_trigger(client, live_server):
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
     set_original()
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_auth.py

@@ -2,13 +2,12 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
+
 
 def test_basic_auth(client, live_server):
 
     live_server_setup(live_server)
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_basicauth_method', _external=True).replace("//","//myuser:mypass@")
@@ -19,7 +18,7 @@ def test_basic_auth(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(1)
+    wait_for_all_checks(client)
 
     # Check form validation
     res = client.post(
@@ -29,7 +28,7 @@ def test_basic_auth(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    time.sleep(1)
+    wait_for_all_checks(client)
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -81,7 +81,7 @@ def test_setup(client, live_server):
 
 # actually only really used by the distll.io importer, but could be handy too
 def test_check_ldjson_price_autodetect(client, live_server):
-
+    #live_server_setup(live_server)
     set_response_with_ldjson()
 
     # Add our URL to the import page
@@ -104,7 +104,6 @@ def test_check_ldjson_price_autodetect(client, live_server):
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
     wait_for_all_checks(client)
 
-    # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     # Offer should be gone
@@ -114,6 +113,9 @@ def test_check_ldjson_price_autodetect(client, live_server):
 
     # and last snapshop (via API) should be just the price
     api_key = extract_api_key_from_UI(client)
+    # Time for writes to happen to history text
+    time.sleep(0.5)
+
     res = client.get(
         url_for("watchsinglehistory", uuid=uuid, timestamp='latest'),
         headers={'x-api-key': api_key},

changedetectionio/tests/test_backend.py

@@ -135,6 +135,9 @@ def test_check_basic_change_detection_functionality(client, live_server):
     # It should have picked up the <title>
     assert b'head title' in res.data
 
+    # Be sure the last_viewed is going to be greater than the last snapshot
+    time.sleep(1)
+
     # hit the mark all viewed link
     res = client.get(url_for("mark_all_viewed"), follow_redirects=True)
 

changedetectionio/tests/test_backup.py

@@ -13,9 +13,6 @@ def test_backup(client, live_server):
 
     set_original_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     res = client.post(
         url_for("import_page"),

changedetectionio/tests/test_block_while_text_present.py

@@ -68,8 +68,6 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
 
     set_original_ignore_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_clone.py

@@ -10,9 +10,6 @@ def test_trigger_functionality(client, live_server):
 
     live_server_setup(live_server)
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     res = client.post(
         url_for("import_page"),

changedetectionio/tests/test_css_selector.py

@@ -77,9 +77,6 @@ def test_check_markup_include_filters_restriction(client, live_server):
 
     set_original_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -89,8 +86,7 @@ def test_check_markup_include_filters_restriction(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -100,22 +96,22 @@ def test_check_markup_include_filters_restriction(client, live_server):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    time.sleep(1)
+    wait_for_all_checks(client)
     # Check it saved
     res = client.get(
         url_for("edit_page", uuid="first"),
     )
     assert bytes(include_filters.encode('utf-8')) in res.data
 
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+
+    wait_for_all_checks(client)
     #  Make a change
     set_modified_response()
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should have 'unviewed' still
     # Because it should be looking at only that 'sametext' id
@@ -138,8 +134,6 @@ def test_check_multiple_filters(client, live_server):
      </html>
     """)
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -149,7 +143,7 @@ def test_check_multiple_filters(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(1)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -164,9 +158,7 @@ def test_check_multiple_filters(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
+    wait_for_all_checks(client)
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True

changedetectionio/tests/test_element_removal.py

@@ -5,7 +5,7 @@ import time
 from flask import url_for
 
 from ..html_tools import *
-from .util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 
 
 def test_setup(live_server):
@@ -111,16 +111,13 @@ def test_element_removal_full(client, live_server):
 
     set_original_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for("test_endpoint", _external=True)
     res = client.post(
         url_for("import_page"), data={"urls": test_url}, follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(1)
+    wait_for_all_checks(client)
     # Goto the edit page, add the filter data
     # Not sure why \r needs to be added - absent of the #changetext this is not necessary
     subtractive_selectors_data = "header\r\nfooter\r\nnav\r\n#changetext"

changedetectionio/tests/test_encoding.py

@@ -27,9 +27,6 @@ def set_html_response():
 def test_check_encoding_detection(client, live_server):
     set_html_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', content_type="text/html", _external=True)
     client.post(
@@ -56,9 +53,6 @@ def test_check_encoding_detection(client, live_server):
 def test_check_encoding_detection_missing_content_type_header(client, live_server):
     set_html_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     client.post(

changedetectionio/tests/test_errorhandling.py

@@ -64,8 +64,6 @@ def test_http_error_handler(client, live_server):
 
 # Just to be sure error text is properly handled
 def test_DNS_errors(client, live_server):
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     # Add our URL to the import page
     res = client.post(
@@ -89,8 +87,6 @@ def test_DNS_errors(client, live_server):
 # Re 1513
 def test_low_level_errors_clear_correctly(client, live_server):
     #live_server_setup(live_server)
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html><body><div id=here>Hello world</div></body></html>")

changedetectionio/tests/test_filter_exist_changes.py

@@ -48,8 +48,6 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
 
     live_server_setup(live_server)
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
     set_response_without_filter()
 
     # Add our URL to the import page

changedetectionio/tests/test_filter_failure_notification.py

@@ -23,8 +23,6 @@ def set_response_with_filter():
 
 def run_filter_test(client, content_filter):
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
     # cleanup for the next
     client.get(
         url_for("form_delete", uuid="all"),
@@ -105,6 +103,10 @@ def run_filter_test(client, content_filter):
     wait_for_all_checks(client)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
+
+    # Give apprise time to fire
+    time.sleep(3)
+
     # Now it should exist and contain our "filter not found" alert
     assert os.path.isfile("test-datastore/notification.txt")
 
@@ -124,6 +126,9 @@ def run_filter_test(client, content_filter):
         client.get(url_for("form_watch_checknow"), follow_redirects=True)
         wait_for_all_checks(client)
 
+    # Give apprise time to fire
+    time.sleep(3)
+
     # It should have sent a notification, but..
     assert os.path.isfile("test-datastore/notification.txt")
     # but it should not contain the info about a failed filter (because there was none in this case)
@@ -149,11 +154,13 @@ def test_setup(live_server):
     live_server_setup(live_server)
 
 def test_check_include_filters_failure_notification(client, live_server):
+    #live_server_setup(live_server)
     set_original_response()
     wait_for_all_checks(client)
     run_filter_test(client, '#nope-doesnt-exist')
 
 def test_check_xpath_filter_failure_notification(client, live_server):
+    # live_server_setup(live_server)
     set_original_response()
     time.sleep(1)
     run_filter_test(client, '//*[@id="nope-doesnt-exist"]')

changedetectionio/tests/test_history_consistency.py

@@ -11,8 +11,6 @@ from urllib.parse import urlparse, parse_qs
 def test_consistent_history(client, live_server):
     live_server_setup(live_server)
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
     r = range(1, 50)
 
     for one in r:

changedetectionio/tests/test_ignorehyperlinks.py

@@ -45,9 +45,6 @@ def test_render_anchor_tag_content_true(client, live_server):
     render_anchor_tag_content setting is set to true"""
     sleep_time_for_fetch_thread = 3
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # set original html text
     set_original_ignore_response()
 

changedetectionio/tests/test_notification_errors.py

@@ -9,9 +9,6 @@ def test_check_notification_error_handling(client, live_server):
     live_server_setup(live_server)
     set_original_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Set a URL and fetch it, then set a notification URL which is going to give errors
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(

changedetectionio/tests/util.py

@@ -120,18 +120,18 @@ def extract_UUID_from_client(client):
     uuid = m.group(1)
     return uuid.strip()
 
-def wait_for_all_checks(client):
-    # Loop waiting until done..
-    attempt=0
-    time.sleep(0.1)
-    while attempt < 60:
-        time.sleep(1)
-        res = client.get(url_for("index"))
-        if not b'Checking now' in res.data:
-            break
-        logging.getLogger().info("Waiting for watch-list to not say 'Checking now'.. {}".format(attempt))
 
-        attempt += 1
+def wait_for_all_checks(client):
+    now = time.time()
+    while time.time() - now <= 30:
+        time.sleep(0.1)
+        p = client.application.view_functions['get_queue_size']()
+        if not p:
+            break
+        logging.getLogger().info(f"Waiting for queue to be empty, queue size {p}  - {time.time() - now}")
+
+    # Empty queue still means that one could be processing, give time for the processing to complete
+    time.sleep(0.2)
 
 def live_server_setup(live_server):
 

changedetectionio/update_worker.py

@@ -160,8 +160,8 @@ class update_worker(threading.Thread):
 
 
     def send_filter_failure_notification(self, watch_uuid):
-
         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
+        logger.trace(f"Watch UUID: {watch_uuid} - Sending filter failure notification - threshold attempts {threshold}")
         watch = self.datastore.data['watching'].get(watch_uuid)
         if not watch:
             return
@@ -335,6 +335,7 @@ class update_worker(threading.Thread):
                         process_changedetection_results = False
 
                     except FilterNotFoundInResponse as e:
+                        logger.debug(f"Watch UUID: {uuid} - Got FilterNotFoundInResponse exception, Consecutive failures - {self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)} - handling..")
                         if not self.datastore.data['watching'].get(uuid):
                             continue
 
@@ -543,4 +544,4 @@ class update_worker(threading.Thread):
                 # Give the CPU time to interrupt
                 time.sleep(0.1)
 
-            self.app.config.exit.wait(1)
+            self.app.config.exit.wait(0.5)

requirements.txt

@@ -1,7 +1,6 @@
 # Used by Pyppeteer
 pyee
-# eventlet 0.33.3 was related to dnspython fixes
-# 0.34.1 - fixes python 3.12 "AttributeError: module 'ssl' has no attribute 'wrap_socket'"
+
 eventlet==0.35.2 # related to dnspython fixes
 feedgen~=0.9
 flask-compress