Test speedup - check janus queues too

.github/dependabot.yml

@@ -4,11 +4,13 @@ updates:
     directory: /
     schedule:
       interval: "weekly"
-    "caronc/apprise":
-      versioning-strategy: "increase"
-      schedule:
-        interval: "daily"
     groups:
       all:
         patterns:
         - "*"
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: "daily"
+    allow:
+      - dependency-name: "apprise"

.github/workflows/containers.yml

@@ -95,7 +95,7 @@ jobs:
           push: true
           tags: |
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:dev,ghcr.io/${{ github.repository }}:dev
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
@@ -133,7 +133,7 @@ jobs:
           file: ./Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max
 # Looks like this was disabled

.github/workflows/test-container-build.yml

@@ -38,8 +38,6 @@ jobs:
             dockerfile: ./Dockerfile
           - platform: linux/arm/v8
             dockerfile: ./Dockerfile
-          - platform: linux/arm64/v8
-            dockerfile: ./Dockerfile
           # Alpine Dockerfile platforms (musl via alpine check)
           - platform: linux/amd64
             dockerfile: ./.github/test/Dockerfile-alpine

changedetectionio/blueprint/watchlist/__init__.py

@@ -87,7 +87,6 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             form=form,
             guid=datastore.data['app_guid'],
             has_proxies=datastore.proxy_list,
-            has_unviewed=datastore.has_unviewed,
             hosted_sticky=os.getenv("SALTED_PASS", False) == False,
             now_time_server=round(time.time()),
             pagination=pagination,
@@ -97,6 +96,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
             system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
             tags=sorted_tags,
+            unread_changes_count=datastore.unread_changes_count,
             watches=sorted_watches
         )
 

changedetectionio/blueprint/watchlist/templates/watch-overview.html

@@ -82,8 +82,11 @@ document.addEventListener('DOMContentLoaded', function() {
         {%- set cols_required = cols_required + 1 -%}
     {%- endif -%}
     {%- set ui_settings = datastore.data['settings']['application']['ui'] -%}
-
-    <div id="watch-table-wrapper">
+    {%- set wrapper_classes = [
+        'has-unread-changes' if unread_changes_count else '',
+        'has-error' if errored_count else '',
+    ] -%}
+    <div id="watch-table-wrapper" class="{{ wrapper_classes | reject('equalto', '') | join(' ') }}">
         {%- set table_classes = [
             'favicon-enabled' if 'favicons_enabled' not in ui_settings or ui_settings['favicons_enabled'] else 'favicon-not-enabled',
         ] -%}
@@ -241,10 +244,10 @@ document.addEventListener('DOMContentLoaded', function() {
             </tbody>
         </table>
         <ul id="post-list-buttons">
-            <li id="post-list-with-errors" class="{%- if errored_count -%}has-error{%- endif -%}" style="display: none;" >
+            <li id="post-list-with-errors" style="display: none;" >
                 <a href="{{url_for('watchlist.index', with_errors=1, tag=request.args.get('tag')) }}" class="pure-button button-tag button-error">With errors ({{ errored_count }})</a>
             </li>
-            <li id="post-list-mark-views" class="{%- if has_unviewed -%}has-unviewed{%- endif -%}" style="display: none;" >
+            <li id="post-list-mark-views" style="display: none;" >
                 <a href="{{url_for('ui.mark_all_viewed',with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag " id="mark-all-viewed">Mark all viewed</a>
             </li>
         {%-  if active_tag_uuid -%}
@@ -252,8 +255,8 @@ document.addEventListener('DOMContentLoaded', function() {
                 <a href="{{url_for('ui.mark_all_viewed', tag=active_tag_uuid) }}" class="pure-button button-tag " id="mark-all-viewed">Mark all viewed in '{{active_tag.title}}'</a>
             </li>
         {%-  endif -%}
-            <li id="post-list-unread" class="{%- if has_unviewed -%}has-unviewed{%- endif -%}" style="display: none;" >
-                <a href="{{url_for('watchlist.index', unread=1, tag=request.args.get('tag')) }}" class="pure-button button-tag">Unread</a>
+            <li id="post-list-unread" style="display: none;" >
+                <a href="{{url_for('watchlist.index', unread=1, tag=request.args.get('tag')) }}" class="pure-button button-tag">Unread (<span id="unread-tab-counter">{{ unread_changes_count }}</span>)</a>
             </li>
             <li>
                <a href="{{ url_for('ui.form_watch_checknow', tag=active_tag_uuid, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag" id="recheck-all">Recheck

changedetectionio/processors/text_json_diff/processor.py

@@ -153,12 +153,26 @@ class perform_site_check(difference_detection_processor):
             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
             self.fetcher.content = html_tools.workarounds_for_obfuscations(self.fetcher.content)
             html_content = self.fetcher.content
+            content_type = self.fetcher.get_all_headers().get('content-type', '').lower()
+            is_attachment = 'attachment' in self.fetcher.get_all_headers().get('content-disposition', '').lower()
 
-            # If not JSON,  and if it's not text/plain..
-            if 'text/plain' in self.fetcher.get_all_headers().get('content-type', '').lower():
+            # Try to detect better mime types if its a download or not announced as HTML
+            if is_attachment or 'octet-stream' in content_type or not 'html' in content_type:
+                logger.debug(f"Got a reply that may be a download or possibly a text attachment, checking..")
+                try:
+                    import magic
+                    mime = magic.from_buffer(html_content, mime=True)
+                    logger.debug(f"Guessing mime type, original content_type '{content_type}', mime type detected '{mime}'")
+                    if mime and "/" in mime: # looks valid and is a valid mime type
+                        content_type = mime
+                except Exception as e:
+                    logger.error(f"Error getting a more precise mime type from 'magic' library ({str(e)}")
+
+            if 'text/' in content_type and not 'html' in content_type:
                 # Don't run get_text or xpath/css filters on plaintext
                 stripped_text_from_html = html_content
             else:
+                # If not JSON, and if it's not text/plain..
                 # Does it have some ld+json price data? used for easier monitoring
                 update_obj['has_ldjson_price_data'] = html_tools.has_ldjson_product_info(self.fetcher.content)
 

changedetectionio/realtime/socket_server.py

@@ -243,14 +243,15 @@ def handle_watch_update(socketio, **kwargs):
 
         general_stats = {
             'count_errors': errored_count,
-            'has_unviewed': datastore.has_unviewed
+            'unread_changes_count': datastore.unread_changes_count
         }
 
         # Debug what's being emitted
         # logger.debug(f"Emitting 'watch_update' event for {watch.get('uuid')}, data: {watch_data}")
 
         # Emit to all clients (no 'broadcast' parameter needed - it's the default behavior)
-        socketio.emit("watch_update", {'watch': watch_data, 'general_stats': general_stats})
+        socketio.emit("watch_update", {'watch': watch_data})
+        socketio.emit("general_stats_update", general_stats)
 
         # Log after successful emit - use watch_data['uuid'] to avoid variable shadowing issues
         logger.trace(f"Socket.IO: Emitted update for watch {watch_data['uuid']}, Checking now: {watch_data['checking_now']}")

changedetectionio/run_socks_proxy_tests.sh

@@ -9,7 +9,7 @@ set -x
 # SOCKS5 related - start simple Socks5 proxy server
 # SOCKSTEST=xyz should show in the logs of this service to confirm it fetched
 docker run --network changedet-network -d --hostname socks5proxy --rm  --name socks5proxy -p 1080:1080 -e PROXY_USER=proxy_user123 -e PROXY_PASSWORD=proxy_pass123 serjs/go-socks5-proxy
-docker run --network changedet-network -d --hostname socks5proxy-noauth --rm  -p 1081:1080 --name socks5proxy-noauth  serjs/go-socks5-proxy
+docker run --network changedet-network -d --hostname socks5proxy-noauth --rm -p 1081:1080 --name socks5proxy-noauth -e REQUIRE_AUTH=false serjs/go-socks5-proxy
 
 echo "---------------------------------- SOCKS5 -------------------"
 # SOCKS5 related - test from proxies.json

changedetectionio/static/js/realtime.js

@@ -117,15 +117,16 @@ $(document).ready(function () {
                 }
             })
 
+            socket.on('general_stats_update', function (general_stats) {
+                // Tabs at bottom of list
+                $('#watch-table-wrapper').toggleClass("has-unread-changes", general_stats.unread_changes_count !==0)
+                $('#watch-table-wrapper').toggleClass("has-error", general_stats.count_errors !== 0)
+                $('#post-list-with-errors a').text(`With errors (${ new Intl.NumberFormat(navigator.language).format(general_stats.count_errors) })`);
+                $('#unread-tab-counter').text(new Intl.NumberFormat(navigator.language).format(general_stats.unread_changes_count));
+            });
+
             socket.on('watch_update', function (data) {
                 const watch = data.watch;
-                const general_stats = data.general_stats;
-
-                // Log the entire watch object for debugging
-                console.log('!!! WATCH UPDATE EVENT RECEIVED !!!');
-                console.log(`${watch.event_timestamp} - Watch update ${watch.uuid} - Checking now - ${watch.checking_now} - UUID in URL ${window.location.href.includes(watch.uuid)}`);
-                console.log('Watch data:', watch);
-                console.log('General stats:', general_stats);
 
                 // Updating watch table rows
                 const $watchRow = $('tr[data-watch-uuid="' + watch.uuid + '"]');
@@ -150,13 +151,6 @@ $(document).ready(function () {
 
                     console.log('Updated UI for watch:', watch.uuid);
                 }
-
-                // Tabs at bottom of list
-                $('#post-list-mark-views').toggleClass("has-unviewed", general_stats.has_unviewed);
-                $('#post-list-unread').toggleClass("has-unviewed", general_stats.has_unviewed);
-                $('#post-list-with-errors').toggleClass("has-error", general_stats.count_errors !== 0)
-                $('#post-list-with-errors a').text(`With errors (${ general_stats.count_errors })`);
-
                 $('body').toggleClass('checking-now', watch.checking_now && window.location.href.includes(watch.uuid));
             });
 

changedetectionio/static/styles/scss/parts/_socket.scss

@@ -17,18 +17,6 @@ body.checking-now {
   position: fixed;
 }
 
-#post-list-buttons {
-  #post-list-with-errors.has-error {
-    display: inline-block !important;
-  }
-  #post-list-mark-views.has-unviewed {
-    display: inline-block !important;
-  }
-  #post-list-unread.has-unviewed {
-    display: inline-block !important;
-  }
-}
-
 
 
 

changedetectionio/static/styles/scss/parts/_watch_table.scss

@@ -127,5 +127,44 @@
       display: inline-block !important;
     }
   }
+
+
 }
 
+#watch-table-wrapper {
+  /* general styling */
+  #post-list-buttons {
+    text-align: right;
+    padding: 0px;
+    margin: 0px;
+
+    li {
+      display: inline-block;
+    }
+
+    a {
+      border-top-left-radius: initial;
+      border-top-right-radius: initial;
+      border-bottom-left-radius: 5px;
+      border-bottom-right-radius: 5px;
+    }
+  }
+
+  /* post list dynamically on/off stuff */
+
+  &.has-error {
+    #post-list-buttons {
+      #post-list-with-errors {
+        display: inline-block !important;
+      }
+    }
+  }
+
+  &.has-unread-changes {
+    #post-list-buttons {
+      #post-list-unread, #post-list-mark-views, #post-list-unread {
+        display: inline-block !important;
+      }
+    }
+  }
+}

changedetectionio/static/styles/scss/styles.scss

@@ -203,24 +203,6 @@ code {
 }
 
 
-#post-list-buttons {
-  text-align: right;
-  padding: 0px;
-  margin: 0px;
-
-  li {
-    display: inline-block;
-  }
-
-  a {
-    border-top-left-radius: initial;
-    border-top-right-radius: initial;
-    border-bottom-left-radius: 5px;
-    border-bottom-right-radius: 5px;
-  }
-}
-
-
 body:after {
   content: "";
   background: linear-gradient(130deg, var(--color-background-gradient-first), var(--color-background-gradient-second) 41.07%, var(--color-background-gradient-third) 84.05%);

changedetectionio/store.py

@@ -202,14 +202,13 @@ class ChangeDetectionStore:
         return seconds
 
     @property
-    def has_unviewed(self):
-        if not self.__data.get('watching'):
-            return None
-
+    def unread_changes_count(self):
+        unread_changes_count = 0
         for uuid, watch in self.__data['watching'].items():
             if watch.history_n >= 2 and watch.viewed == False:
-                return True
-        return False
+                unread_changes_count += 1
+
+        return unread_changes_count
 
     @property
     def data(self):

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -75,7 +75,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     wait_for_all_checks(client)
     time.sleep(0.5)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # The trigger line is REMOVED,  this should trigger
     set_original(excluding='The golden line')
@@ -84,7 +84,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     time.sleep(1)
 
@@ -98,14 +98,14 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     wait_for_all_checks(client)
     time.sleep(1)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # Remove it again, and we should get a trigger
     set_original(excluding='The golden line')
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
@@ -169,7 +169,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
 
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # The trigger line is ADDED,  this should trigger
     set_original(add_line='<p>Oh yes please</p>')
@@ -177,7 +177,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
 
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     # Takes a moment for apprise to fire
     wait_for_notification_endpoint_output()

changedetectionio/tests/test_backend.py

@@ -38,9 +38,9 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
         # Give the thread time to pick it up
         wait_for_all_checks(client)
 
-        # It should report nothing found (no new 'unviewed' class)
+        # It should report nothing found (no new 'has-unread-changes' class)
         res = client.get(url_for("watchlist.index"))
-        assert b'unviewed' not in res.data
+        assert b'has-unread-changes' not in res.data
         assert b'test-endpoint' in res.data
 
         # Default no password set, this stuff should be always available.
@@ -74,9 +74,9 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     res = client.get(url_for("ui.ui_edit.watch_get_latest_html", uuid=uuid))
     assert b'which has this one new line' in res.data
 
-    # Now something should be ready, indicated by having a 'unviewed' class
+    # Now something should be ready, indicated by having a 'has-unread-changes' class
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     # #75, and it should be in the RSS feed
     rss_token = extract_rss_token_from_UI(client)
@@ -90,7 +90,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     assert expected_url.encode('utf-8') in res.data
 #
-    # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times
+    # Following the 'diff' link, it should no longer display as 'has-unread-changes' even after we recheck it a few times
     res = client.get(url_for("ui.ui_views.diff_history_page", uuid=uuid))
     assert b'selected=""' in res.data, "Confirm diff history page loaded"
 
@@ -111,12 +111,12 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
         # Give the thread time to pick it up
         wait_for_all_checks(client)
 
-        # It should report nothing found (no new 'unviewed' class)
+        # It should report nothing found (no new 'has-unread-changes' class)
         res = client.get(url_for("watchlist.index"))
 
 
-        assert b'unviewed' not in res.data
-        assert b'class="has-unviewed' not in res.data
+        assert b'has-unread-changes' not in res.data
+        assert b'class="has-unread-changes' not in res.data
         assert b'head title' in res.data  # Should be ON by default
         assert b'test-endpoint' in res.data
 
@@ -140,8 +140,8 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     wait_for_all_checks(client)
 
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
-    assert b'class="has-unviewed' in res.data
+    assert b'has-unread-changes' in res.data
+    assert b'class="has-unread-changes' in res.data
     assert b'head title' not in res.data  # should now be off
 
 
@@ -151,8 +151,8 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # hit the mark all viewed link
     res = client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
 
-    assert b'class="has-unviewed' not in res.data
-    assert b'unviewed' not in res.data
+    assert b'class="has-unread-changes' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # #2458 "clear history" should make the Watch object update its status correctly when the first snapshot lands again
     client.get(url_for("ui.clear_watch_history", uuid=uuid))
@@ -165,3 +165,53 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # Cleanup everything
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
+
+def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage):
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("""some random text that should be split by line
+and not parsed with html_to_text
+this way we know that it correctly parsed as plain text
+\r\n
+ok\r\n
+got it\r\n
+""")
+
+    test_url = url_for('test_endpoint', content_type="application/octet-stream", _external=True)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("imports.import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    wait_for_all_checks(client)
+
+    ### check the front end
+    res = client.get(
+        url_for("ui.ui_views.preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"some random text that should be split by line\n" in res.data
+    ####
+
+    # Check the snapshot by API that it has linefeeds too
+    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+    res = client.get(
+        url_for("watchhistory", uuid=watch_uuid),
+        headers={'x-api-key': api_key},
+    )
+
+    # Fetch a snapshot by timestamp, check the right one was found
+    res = client.get(
+        url_for("watchsinglehistory", uuid=watch_uuid, timestamp=list(res.json.keys())[-1]),
+        headers={'x-api-key': api_key},
+    )
+    assert b"some random text that should be split by line\n" in res.data
+
+
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
+

changedetectionio/tests/test_basic_socketio.py

@@ -58,6 +58,7 @@ def run_socketio_watch_update_test(client, live_server, password_mode=""):
 
     has_watch_update = False
     has_unviewed_update = False
+    got_general_stats_update = False
 
     for i in range(10):
         # Get received events
@@ -65,15 +66,11 @@ def run_socketio_watch_update_test(client, live_server, password_mode=""):
 
         if received:
             logger.info(f"Received {len(received)} events after {i+1} seconds")
-
-            # Check for watch_update events with unviewed=True
             for event in received:
                 if event['name'] == 'watch_update':
                     has_watch_update = True
-                    if event['args'][0]['watch'].get('unviewed', False):
-                        has_unviewed_update = True
-                        logger.info("Found unviewed update event!")
-                        break
+                if event['name'] == 'general_stats_update':
+                    got_general_stats_update = True
 
         if has_unviewed_update:
             break
@@ -92,7 +89,7 @@ def run_socketio_watch_update_test(client, live_server, password_mode=""):
     assert has_watch_update, "No watch_update events received"
 
     # Verify we received an unviewed event
-    assert has_unviewed_update, "No watch_update event with unviewed=True received"
+    assert got_general_stats_update, "Got general stats update event"
 
     # Alternatively, check directly if the watch in the datastore is marked as unviewed
     from changedetectionio.flask_app import app

changedetectionio/tests/test_block_while_text_present.py

@@ -107,9 +107,9 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert b'/test-endpoint' in res.data
 
     # The page changed, BUT the text is still there, just the rest of it changes, we should not see a change
@@ -120,9 +120,9 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert b'/test-endpoint' in res.data
 
     # 2548
@@ -131,7 +131,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
 
     # Now we set a change where the text is gone AND its different content, it should now trigger
@@ -139,7 +139,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
 
 

changedetectionio/tests/test_conditions.py

@@ -125,7 +125,7 @@ def test_conditions_with_text_and_number(client, live_server):
     time.sleep(2)
     # 75 is > 20 and < 100 and contains "5"
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
 
     # Case 2: Change with one condition violated
@@ -141,7 +141,7 @@ def test_conditions_with_text_and_number(client, live_server):
 
     # Should NOT be marked as having changes since not all conditions are met
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
@@ -299,7 +299,7 @@ def test_lev_conditions_plugin(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # Check the content saved initially, even tho a condition was set - this is the first snapshot so shouldnt be affected by conditions
     res = client.get(
@@ -326,7 +326,7 @@ def test_lev_conditions_plugin(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data #because this will be like 0.90 not 0.8 threshold
+    assert b'has-unread-changes' not in res.data #because this will be like 0.90 not 0.8 threshold
 
     ############### Now change it a MORE THAN 50%
     test_return_data = """<html>
@@ -345,7 +345,7 @@ def test_lev_conditions_plugin(client, live_server, measure_memory_usage):
     assert b'Queued 1 watch for rechecking.' in res.data
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
     # cleanup for the next
     client.get(
         url_for("ui.form_delete", uuid="all"),

changedetectionio/tests/test_css_selector.py

@@ -116,10 +116,10 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
-    # It should have 'unviewed' still
+    # It should have 'has-unread-changes' still
     # Because it should be looking at only that 'sametext' id
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
 
 # Tests the whole stack works with the CSS Filter

changedetectionio/tests/test_element_removal.py

@@ -190,7 +190,7 @@ def test_element_removal_full(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
 
-    # so that we set the state to 'unviewed' after all the edits
+    # so that we set the state to 'has-unread-changes' after all the edits
     client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
 
     #  Make a change to header/footer/nav

changedetectionio/tests/test_errorhandling.py

@@ -31,7 +31,7 @@ def _runner_test_http_errors(client, live_server, http_code, expected_text):
 
     res = client.get(url_for("watchlist.index"))
     # no change
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert bytes(expected_text.encode('utf-8')) in res.data
 
 

changedetectionio/tests/test_extract_regex.py

@@ -174,10 +174,10 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should have 'unviewed' still
+    # It should have 'has-unread-changes' still
     # Because it should be looking at only that 'sametext' id
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     # Check HTML conversion detected and workd
     res = client.get(

changedetectionio/tests/test_ignore_text.py

@@ -128,9 +128,9 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert b'/test-endpoint' in res.data
 
     #  Make a change
@@ -141,9 +141,9 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert b'/test-endpoint' in res.data
 
 
@@ -154,7 +154,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     wait_for_all_checks(client)
 
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     res = client.get(url_for("ui.ui_views.preview_page", uuid="first"))
 
@@ -222,9 +222,9 @@ def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    # It should report nothing found (no new 'unviewed' class), adding random ignore text should not cause a change
+    # It should report nothing found (no new 'has-unread-changes' class), adding random ignore text should not cause a change
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert b'/test-endpoint' in res.data
 #####
 
@@ -238,10 +238,10 @@ def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
 
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert b'/test-endpoint' in res.data
 
     # Just to be sure.. set a regular modified change that will trigger it
@@ -249,7 +249,7 @@ def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_ignorehyperlinks.py

@@ -111,7 +111,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag
     assert '(/modified_link)' in res.data.decode()
 
     # since the link has changed, and we chose to render anchor tag content,
-    # we should detect a change (new 'unviewed' class)
+    # we should detect a change (new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
     assert b"unviewed" in res.data
     assert b"/test-endpoint" in res.data

changedetectionio/tests/test_ignorestatuscode.py

@@ -77,9 +77,9 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server, me
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
     assert b'/test-endpoint' in res.data
 
 
@@ -124,8 +124,8 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server, measu
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should have 'unviewed' still
+    # It should have 'has-unread-changes' still
     # Because it should be looking at only that 'sametext' id
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 

changedetectionio/tests/test_ignorewhitespace.py

@@ -89,7 +89,7 @@ def test_check_ignore_whitespace(client, live_server, measure_memory_usage):
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert b'/test-endpoint' in res.data

changedetectionio/tests/test_jinja2.py

@@ -26,7 +26,7 @@ def test_jinja2_in_url_query(client, live_server, measure_memory_usage):
     assert b"Watch added" in res.data
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(
         url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
@@ -51,7 +51,7 @@ def test_jinja2_security_url_query(client, live_server, measure_memory_usage):
     assert b"Watch added" in res.data
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
     assert b'is invalid and cannot be used' in res.data
     # Some of the spewed output from the subclasses

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -280,9 +280,9 @@ def check_json_filter(json_filter, client, live_server):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should have 'unviewed' still
+    # It should have 'has-unread-changes' still
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     # Should not see this, because its not in the JSONPath we entered
     res = client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
@@ -418,14 +418,14 @@ def check_json_ext_filter(json_filter, client, live_server):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should have 'unviewed'
+    # It should have 'has-unread-changes'
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     res = client.get(url_for("ui.ui_views.preview_page", uuid="first"))
 
     # We should never see 'ForSale' because we are selecting on 'Sold' in the rule,
-    # But we should know it triggered ('unviewed' assert above)
+    # But we should know it triggered ('has-unread-changes' assert above)
     assert b'ForSale' not in res.data
     assert b'Sold' in res.data
 
@@ -465,7 +465,7 @@ def test_ignore_json_order(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # Just to be sure it still works
     with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -476,7 +476,7 @@ def test_ignore_json_order(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_nonrenderable_pages.py

@@ -40,9 +40,9 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
 
     #####################
@@ -62,9 +62,9 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
@@ -92,9 +92,9 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
     client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
     time.sleep(0.2)
 
@@ -108,7 +108,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data # A change should have registered because empty_pages_are_a_change is ON
+    assert b'has-unread-changes' in res.data # A change should have registered because empty_pages_are_a_change is ON
     assert b'fetch-error' not in res.data
 
     #

changedetectionio/tests/test_pdf.py

@@ -49,9 +49,9 @@ def test_fetch_pdf(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
 
-    # Now something should be ready, indicated by having a 'unviewed' class
+    # Now something should be ready, indicated by having a 'has-unread-changes' class
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     # The original checksum should be not be here anymore (cdio adds it to the bottom of the text)
 

changedetectionio/tests/test_preview_endpoints.py

@@ -47,9 +47,9 @@ def test_fetch_pdf(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
 
-    # Now something should be ready, indicated by having a 'unviewed' class
+    # Now something should be ready, indicated by having a 'has-unread-changes' class
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     # The original checksum should be not be here anymore (cdio adds it to the bottom of the text)
 

changedetectionio/tests/test_restock_itemprop.py

@@ -112,7 +112,7 @@ def test_itemprop_price_change(client, live_server):
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'180.45' in res.data
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
     client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
     time.sleep(0.2)
 
@@ -129,7 +129,7 @@ def test_itemprop_price_change(client, live_server):
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'120.45' in res.data
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
@@ -178,7 +178,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
     assert b'more than one price detected' not in res.data
     # BUT the new price should show, even tho its within limits
     assert b'1,000.45' or b'1000.45' in res.data #depending on locale
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # price changed to something LESS than min (900), SHOULD be a change
     set_original_response(props_markup=instock_props[0], price='890.45')
@@ -188,7 +188,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'890.45' in res.data
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     client.get(url_for("ui.mark_all_viewed"))
 
@@ -200,7 +200,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'820.45' in res.data
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
     client.get(url_for("ui.mark_all_viewed"))
 
     # price changed to something MORE than max (1100.10), SHOULD be a change
@@ -210,7 +210,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
     res = client.get(url_for("watchlist.index"))
     # Depending on the LOCALE it may be either of these (generally for US/default/etc)
     assert b'1,890.45' in res.data or b'1890.45' in res.data
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
@@ -294,7 +294,7 @@ def test_itemprop_percent_threshold(client, live_server):
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'960.45' in res.data
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # Bigger INCREASE change than the threshold should trigger
     set_original_response(props_markup=instock_props[0], price='1960.45')
@@ -302,7 +302,7 @@ def test_itemprop_percent_threshold(client, live_server):
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'1,960.45' or b'1960.45' in res.data #depending on locale
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
 
     # Small decrease should NOT trigger
@@ -312,7 +312,7 @@ def test_itemprop_percent_threshold(client, live_server):
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'1,950.45' or b'1950.45' in res.data #depending on locale
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
 
 

changedetectionio/tests/test_source.py

@@ -43,9 +43,9 @@ def test_check_basic_change_detection_functionality_source(client, live_server,
 
     wait_for_all_checks(client)
 
-    # Now something should be ready, indicated by having a 'unviewed' class
+    # Now something should be ready, indicated by having a 'has-unread-changes' class
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     res = client.get(
         url_for("ui.ui_views.diff_history_page", uuid="first"),

changedetectionio/tests/test_trigger.py

@@ -96,7 +96,7 @@ def test_trigger_functionality(client, live_server, measure_memory_usage):
 
 
     
-    # so that we set the state to 'unviewed' after all the edits
+    # so that we set the state to 'has-unread-changes' after all the edits
     client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
 
     # Trigger a check
@@ -104,9 +104,9 @@ def test_trigger_functionality(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert b'/test-endpoint' in res.data
 
     #  Make a change
@@ -116,9 +116,9 @@ def test_trigger_functionality(client, live_server, measure_memory_usage):
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # Now set the content which contains the trigger text
     set_modified_with_trigger_text_response()
@@ -126,7 +126,7 @@ def test_trigger_functionality(client, live_server, measure_memory_usage):
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
     
     # https://github.com/dgtlmoon/changedetection.io/issues/616
     # Apparently the actual snapshot that contains the trigger never shows

changedetectionio/tests/test_trigger_regex.py

@@ -42,7 +42,7 @@ def test_trigger_regex_functionality(client, live_server, measure_memory_usage):
 
     # It should report nothing found (just a new one shouldnt have anything)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     ### test regex
     res = client.post(
@@ -54,7 +54,7 @@ def test_trigger_regex_functionality(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     wait_for_all_checks(client)
-    # so that we set the state to 'unviewed' after all the edits
+    # so that we set the state to 'has-unread-changes' after all the edits
     client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -65,7 +65,7 @@ def test_trigger_regex_functionality(client, live_server, measure_memory_usage):
 
     # It should report nothing found (nothing should match the regex)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("regex test123<br>\nsomething 123")
@@ -73,7 +73,7 @@ def test_trigger_regex_functionality(client, live_server, measure_memory_usage):
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     # Cleanup everything
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_trigger_regex_with_filter.py

@@ -69,7 +69,7 @@ def test_trigger_regex_functionality_with_filter(client, live_server, measure_me
 
     # It should report nothing found (nothing should match the regex and filter)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # now this should trigger something
     with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -78,7 +78,7 @@ def test_trigger_regex_functionality_with_filter(client, live_server, measure_me
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
 # Cleanup everything
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_ui.py

@@ -248,3 +248,44 @@ def test_page_title_listing_behaviour(client, live_server):
     res = client.get(url_for("watchlist.index"))
     assert b"head titlecustom html" in res.data
 
+
+def test_ui_viewed_unread_flag(client, live_server):
+
+    import time
+
+    set_original_response(extra_title="custom html")
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("imports.import_page"),
+        data={"urls": url_for('test_endpoint', _external=True)+"\r\n"+url_for('test_endpoint', _external=True)},
+        follow_redirects=True
+    )
+
+    assert b"2 Imported" in res.data
+    wait_for_all_checks(client)
+
+    set_modified_response()
+    res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    assert b'Queued 2 watches for rechecking.' in res.data
+    wait_for_all_checks(client)
+    res = client.get(url_for("watchlist.index"))
+    assert b'<span id="unread-tab-counter">2</span>' in res.data
+    assert res.data.count(b'data-watch-uuid') == 2
+
+    # one should now be viewed, but two in total still
+    client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
+    res = client.get(url_for("watchlist.index"))
+    assert b'<span id="unread-tab-counter">1</span>' in res.data
+    assert res.data.count(b'data-watch-uuid') == 2
+
+    # check ?unread=1 works
+    res = client.get(url_for("watchlist.index")+"?unread=1")
+    assert res.data.count(b'data-watch-uuid') == 1
+    assert b'<span id="unread-tab-counter">1</span>' in res.data
+
+    # Mark all viewed test again
+    client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
+    time.sleep(0.2)
+    res = client.get(url_for("watchlist.index"))
+    assert b'<span id="unread-tab-counter">0</span>' in res.data

changedetectionio/tests/test_unique_lines.py

@@ -97,7 +97,7 @@ def test_unique_lines_functionality(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     #  Make a change
     set_modified_swapped_lines()
@@ -108,16 +108,16 @@ def test_unique_lines_functionality(client, live_server, measure_memory_usage):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # Now set the content which contains the new text and re-ordered existing text
     set_modified_with_trigger_text_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
@@ -157,7 +157,7 @@ def test_sort_lines_functionality(client, live_server, measure_memory_usage):
 
     res = client.get(url_for("watchlist.index"))
     # Should be a change registered
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     res = client.get(
         url_for("ui.ui_views.preview_page", uuid="first"),

changedetectionio/tests/test_xpath_selector.py

@@ -208,7 +208,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server, measure_memo
     wait_for_all_checks(client)
 
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 

changedetectionio/tests/util.py

@@ -130,40 +130,47 @@ def extract_UUID_from_client(client):
 
 def wait_for_all_checks(client=None):
     """
-    Waits until the queue is empty and workers are idle.
-    Much faster than the original with adaptive timing.
+    Waits until both queues are empty and all workers are idle.
+    Optimized for Janus queues with minimal delays.
     """
-    from changedetectionio.flask_app import update_q as global_update_q
+    from changedetectionio.flask_app import update_q as global_update_q, notification_q
     from changedetectionio import worker_handler
 
     logger = logging.getLogger()
-    empty_since = None
-    attempt = 0
-    max_attempts = 150  # Still reasonable upper bound
 
-    while attempt < max_attempts:
-        # Start with fast checks, slow down if needed
-        if attempt < 10:
-            time.sleep(0.1)  # Very fast initial checks
-        elif attempt < 30:
-            time.sleep(0.3)  # Medium speed
-        else:
-            time.sleep(0.8)  # Slower for persistent issues
+    # Much tighter timing with reliable Janus queues
+    max_attempts = 100  # Reduced from 150
 
-        q_length = global_update_q.qsize()
+    for attempt in range(max_attempts):
+        # Check both queues and worker status
+        update_q_size = global_update_q.qsize()
+        notification_q_size = notification_q.qsize()
         running_uuids = worker_handler.get_running_uuids()
         any_workers_busy = len(running_uuids) > 0
 
-        if q_length == 0 and not any_workers_busy:
-            if empty_since is None:
-                empty_since = time.time()
-            elif time.time() - empty_since >= 0.15:  # Shorter wait
-                break
+        # Both queues empty and no workers processing
+        if update_q_size == 0 and notification_q_size == 0 and not any_workers_busy:
+            # Small delay to account for items being added to queue during processing
+            time.sleep(0.05)
+
+            # Double-check after brief delay
+            update_q_size = global_update_q.qsize()
+            notification_q_size = notification_q.qsize()
+            running_uuids = worker_handler.get_running_uuids()
+            any_workers_busy = len(running_uuids) > 0
+
+            if update_q_size == 0 and notification_q_size == 0 and not any_workers_busy:
+                return  # All clear!
+
+        # Adaptive sleep timing - start fast, get slightly slower
+        if attempt < 20:
+            time.sleep(0.05)  # Very fast initial checks
+        elif attempt < 50:
+            time.sleep(0.1)   # Medium speed
         else:
-            empty_since = None
-        
-        attempt += 1
-        time.sleep(0.3)
+            time.sleep(0.2)   # Slower for edge cases
+
+    logger.warning(f"wait_for_all_checks() timed out after {max_attempts} attempts")
 
 # Replaced by new_live_server_setup and calling per function scope in conftest.py
 def  live_server_setup(live_server):

requirements.txt

@@ -39,7 +39,7 @@ jsonpath-ng~=1.5.3
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise==1.9.3
+apprise==1.9.4
 
 # - Needed for apprise/spush, and maybe others? hopefully doesnt trigger a rust compile.
 # - Requires extra wheel for rPi, adds build time for arm/v8 which is not in piwheels
@@ -51,8 +51,8 @@ cryptography==44.0.1
 # use any version other than 2.0.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
 paho-mqtt!=2.0.*
 
-# Used for CSS filtering
-beautifulsoup4>=4.0.0
+# Used for CSS filtering, JSON extraction from HTML
+beautifulsoup4>=4.0.0,<=4.13.5
 
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
 # #2328 - 5.2.0 and 5.2.1 had extra CPU flag CFLAGS set which was not compatible on older hardware