fix logic

changedetectionio/flask_app.py

@@ -45,6 +45,7 @@ running_update_threads = []
 ticker_thread = None
 
 extra_stylesheets = []
+
 update_q = queue.PriorityQueue()
 notification_q = queue.Queue()
 
@@ -1537,11 +1538,6 @@ def changedetection_app(config=None, datastore_o=None):
         # paste in etc
         return redirect(url_for('index'))
 
-    @app.route("/queue_size", methods=['GET'])
-    @login_optionally_required
-    def get_queue_size():
-        return update_q.qsize()
-
     @app.route("/highlight_submit_ignore_url", methods=['POST'])
     @login_optionally_required
     def highlight_submit_ignore_url():

changedetectionio/model/Watch.py

@@ -217,8 +217,6 @@ class model(dict):
         fname = os.path.join(self.watch_data_dir, "history.txt")
         if os.path.isfile(fname):
             logger.debug(f"Reading watch history index for {self.get('uuid')}")
-
-
             with open(fname, "r") as f:
                 for i in f.readlines():
                     if ',' in i:
@@ -335,9 +333,7 @@ class model(dict):
         # Small hack so that we sleep just enough to allow 1 second  between history snapshots
         # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
         if self.__newest_history_key and int(timestamp) == int(self.__newest_history_key):
-            logger.warning(f"Timestamp {timestamp} already exists, waiting 1 seconds so we have a unique key in history.txt")
-            timestamp = str(int(timestamp) + 1)
-            time.sleep(1)
+            time.sleep(timestamp - self.__newest_history_key)
 
         threshold = int(os.getenv('SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD', 1024))
         skip_brotli = strtobool(os.getenv('DISABLE_BROTLI_TEXT_SNAPSHOT', 'False'))

changedetectionio/store.py

@@ -178,7 +178,7 @@ class ChangeDetectionStore:
     @property
     def has_unviewed(self):
         for uuid, watch in self.__data['watching'].items():
-            if watch.history_n >= 2 and watch.viewed == False:
+            if watch.viewed == False:
                 return True
         return False
 

changedetectionio/templates/watch-overview.html

@@ -78,7 +78,7 @@
             <tbody>
             {% if not watches|length %}
             <tr>
-                <td colspan="6" style="text-wrap: wrap;">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('import_page')}}" >import a list</a>.</td>
+                <td colspan="6">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('import_page')}}" >import a list</a>.</td>
             </tr>
             {% endif %}
             {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) %}

changedetectionio/tests/restock/test_restock.py

@@ -53,6 +53,7 @@ def test_restock_detection(client, live_server):
     set_original_response()
     #assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
 
+    time.sleep(1)
     live_server_setup(live_server)
     #####################
     notification_url = url_for('test_notification_endpoint', _external=True).replace('http://localhost', 'http://changedet').replace('http', 'json')

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -71,8 +71,6 @@ def test_check_notification_email_formats_default_HTML(client, live_server):
 
     wait_for_all_checks(client)
     set_longer_modified_response()
-    time.sleep(2)
-
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -137,7 +135,6 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
     wait_for_all_checks(client)
     set_longer_modified_response()
-    time.sleep(2)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -40,6 +40,8 @@ def test_setup(client, live_server):
 
 def test_check_removed_line_contains_trigger(client, live_server):
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
     set_original()
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_auth.py

@@ -2,12 +2,13 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
-
+from . util import live_server_setup
 
 def test_basic_auth(client, live_server):
 
     live_server_setup(live_server)
+    # Give the endpoint time to spin up
+    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_basicauth_method', _external=True).replace("//","//myuser:mypass@")
@@ -18,7 +19,7 @@ def test_basic_auth(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
+    time.sleep(1)
 
     # Check form validation
     res = client.post(
@@ -28,7 +29,7 @@ def test_basic_auth(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(1)
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -81,7 +81,7 @@ def test_setup(client, live_server):
 
 # actually only really used by the distll.io importer, but could be handy too
 def test_check_ldjson_price_autodetect(client, live_server):
-    #live_server_setup(live_server)
+
     set_response_with_ldjson()
 
     # Add our URL to the import page
@@ -104,6 +104,7 @@ def test_check_ldjson_price_autodetect(client, live_server):
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
     wait_for_all_checks(client)
 
+    # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     # Offer should be gone
@@ -113,9 +114,6 @@ def test_check_ldjson_price_autodetect(client, live_server):
 
     # and last snapshop (via API) should be just the price
     api_key = extract_api_key_from_UI(client)
-    # Time for writes to happen to history text
-    time.sleep(0.5)
-
     res = client.get(
         url_for("watchsinglehistory", uuid=uuid, timestamp='latest'),
         headers={'x-api-key': api_key},

changedetectionio/tests/test_backend.py

@@ -135,9 +135,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
     # It should have picked up the <title>
     assert b'head title' in res.data
 
-    # Be sure the last_viewed is going to be greater than the last snapshot
-    time.sleep(1)
-
     # hit the mark all viewed link
     res = client.get(url_for("mark_all_viewed"), follow_redirects=True)
 

changedetectionio/tests/test_backup.py

@@ -13,6 +13,9 @@ def test_backup(client, live_server):
 
     set_original_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Add our URL to the import page
     res = client.post(
         url_for("import_page"),

changedetectionio/tests/test_block_while_text_present.py

@@ -68,6 +68,8 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
 
     set_original_ignore_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_clone.py

@@ -10,6 +10,9 @@ def test_trigger_functionality(client, live_server):
 
     live_server_setup(live_server)
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Add our URL to the import page
     res = client.post(
         url_for("import_page"),

changedetectionio/tests/test_css_selector.py

@@ -77,6 +77,9 @@ def test_check_markup_include_filters_restriction(client, live_server):
 
     set_original_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -86,7 +89,8 @@ def test_check_markup_include_filters_restriction(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -96,22 +100,22 @@ def test_check_markup_include_filters_restriction(client, live_server):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
+    time.sleep(1)
     # Check it saved
     res = client.get(
         url_for("edit_page", uuid="first"),
     )
     assert bytes(include_filters.encode('utf-8')) in res.data
 
-
-    wait_for_all_checks(client)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
     #  Make a change
     set_modified_response()
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should have 'unviewed' still
     # Because it should be looking at only that 'sametext' id
@@ -134,6 +138,8 @@ def test_check_multiple_filters(client, live_server):
      </html>
     """)
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -143,7 +149,7 @@ def test_check_multiple_filters(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
+    time.sleep(1)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -158,7 +164,9 @@ def test_check_multiple_filters(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    wait_for_all_checks(client)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True

changedetectionio/tests/test_element_removal.py

@@ -5,7 +5,7 @@ import time
 from flask import url_for
 
 from ..html_tools import *
-from .util import live_server_setup, wait_for_all_checks
+from .util import live_server_setup
 
 
 def test_setup(live_server):
@@ -111,13 +111,16 @@ def test_element_removal_full(client, live_server):
 
     set_original_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Add our URL to the import page
     test_url = url_for("test_endpoint", _external=True)
     res = client.post(
         url_for("import_page"), data={"urls": test_url}, follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
+    time.sleep(1)
     # Goto the edit page, add the filter data
     # Not sure why \r needs to be added - absent of the #changetext this is not necessary
     subtractive_selectors_data = "header\r\nfooter\r\nnav\r\n#changetext"

changedetectionio/tests/test_encoding.py

@@ -27,6 +27,9 @@ def set_html_response():
 def test_check_encoding_detection(client, live_server):
     set_html_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Add our URL to the import page
     test_url = url_for('test_endpoint', content_type="text/html", _external=True)
     client.post(
@@ -53,6 +56,9 @@ def test_check_encoding_detection(client, live_server):
 def test_check_encoding_detection_missing_content_type_header(client, live_server):
     set_html_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     client.post(

changedetectionio/tests/test_errorhandling.py

@@ -64,6 +64,8 @@ def test_http_error_handler(client, live_server):
 
 # Just to be sure error text is properly handled
 def test_DNS_errors(client, live_server):
+    # Give the endpoint time to spin up
+    time.sleep(1)
 
     # Add our URL to the import page
     res = client.post(
@@ -87,6 +89,8 @@ def test_DNS_errors(client, live_server):
 # Re 1513
 def test_low_level_errors_clear_correctly(client, live_server):
     #live_server_setup(live_server)
+    # Give the endpoint time to spin up
+    time.sleep(1)
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html><body><div id=here>Hello world</div></body></html>")

changedetectionio/tests/test_filter_exist_changes.py

@@ -48,6 +48,8 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
 
     live_server_setup(live_server)
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
     set_response_without_filter()
 
     # Add our URL to the import page

changedetectionio/tests/test_filter_failure_notification.py

@@ -23,6 +23,8 @@ def set_response_with_filter():
 
 def run_filter_test(client, content_filter):
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
     # cleanup for the next
     client.get(
         url_for("form_delete", uuid="all"),
@@ -103,10 +105,6 @@ def run_filter_test(client, content_filter):
     wait_for_all_checks(client)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-
-    # Give apprise time to fire
-    time.sleep(3)
-
     # Now it should exist and contain our "filter not found" alert
     assert os.path.isfile("test-datastore/notification.txt")
 
@@ -126,9 +124,6 @@ def run_filter_test(client, content_filter):
         client.get(url_for("form_watch_checknow"), follow_redirects=True)
         wait_for_all_checks(client)
 
-    # Give apprise time to fire
-    time.sleep(3)
-
     # It should have sent a notification, but..
     assert os.path.isfile("test-datastore/notification.txt")
     # but it should not contain the info about a failed filter (because there was none in this case)
@@ -154,13 +149,11 @@ def test_setup(live_server):
     live_server_setup(live_server)
 
 def test_check_include_filters_failure_notification(client, live_server):
-    #live_server_setup(live_server)
     set_original_response()
     wait_for_all_checks(client)
     run_filter_test(client, '#nope-doesnt-exist')
 
 def test_check_xpath_filter_failure_notification(client, live_server):
-    # live_server_setup(live_server)
     set_original_response()
     time.sleep(1)
     run_filter_test(client, '//*[@id="nope-doesnt-exist"]')

changedetectionio/tests/test_history_consistency.py

@@ -11,6 +11,8 @@ from urllib.parse import urlparse, parse_qs
 def test_consistent_history(client, live_server):
     live_server_setup(live_server)
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
     r = range(1, 50)
 
     for one in r:

changedetectionio/tests/test_ignorehyperlinks.py

@@ -45,6 +45,9 @@ def test_render_anchor_tag_content_true(client, live_server):
     render_anchor_tag_content setting is set to true"""
     sleep_time_for_fetch_thread = 3
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # set original html text
     set_original_ignore_response()
 

changedetectionio/tests/test_notification_errors.py

@@ -9,6 +9,9 @@ def test_check_notification_error_handling(client, live_server):
     live_server_setup(live_server)
     set_original_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Set a URL and fetch it, then set a notification URL which is going to give errors
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(

changedetectionio/tests/util.py

@@ -120,18 +120,18 @@ def extract_UUID_from_client(client):
     uuid = m.group(1)
     return uuid.strip()
 
-
 def wait_for_all_checks(client):
-    now = time.time()
-    while time.time() - now <= 30:
-        time.sleep(0.1)
-        p = client.application.view_functions['get_queue_size']()
-        if not p:
+    # Loop waiting until done..
+    attempt=0
+    time.sleep(0.1)
+    while attempt < 60:
+        time.sleep(1)
+        res = client.get(url_for("index"))
+        if not b'Checking now' in res.data:
             break
-        logging.getLogger().info(f"Waiting for queue to be empty, queue size {p}  - {time.time() - now}")
+        logging.getLogger().info("Waiting for watch-list to not say 'Checking now'.. {}".format(attempt))
 
-    # Empty queue still means that one could be processing, give time for the processing to complete
-    time.sleep(0.2)
+        attempt += 1
 
 def live_server_setup(live_server):
 

changedetectionio/update_worker.py

@@ -160,8 +160,8 @@ class update_worker(threading.Thread):
 
 
     def send_filter_failure_notification(self, watch_uuid):
+
         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
-        logger.trace(f"Watch UUID: {watch_uuid} - Sending filter failure notification - threshold attempts {threshold}")
         watch = self.datastore.data['watching'].get(watch_uuid)
         if not watch:
             return
@@ -335,7 +335,6 @@ class update_worker(threading.Thread):
                         process_changedetection_results = False
 
                     except FilterNotFoundInResponse as e:
-                        logger.debug(f"Watch UUID: {uuid} - Got FilterNotFoundInResponse exception, Consecutive failures - {self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)} - handling..")
                         if not self.datastore.data['watching'].get(uuid):
                             continue
 
@@ -544,4 +543,4 @@ class update_worker(threading.Thread):
                 # Give the CPU time to interrupt
                 time.sleep(0.1)
 
-            self.app.config.exit.wait(0.5)
+            self.app.config.exit.wait(1)

requirements.txt

@@ -1,7 +1,7 @@
 # Used by Pyppeteer
 pyee
 
-eventlet==0.35.2 # related to dnspython fixes
+eventlet==0.33.3 # related to dnspython fixes
 feedgen~=0.9
 flask-compress
 # 0.6.3 included compatibility fix for werkzeug 3.x (2.x had deprecation of url handlers)