Test speedup - check janus queues too

.github/dependabot.yml

@@ -4,11 +4,13 @@ updates:
     directory: /
     schedule:
       interval: "weekly"
-    "caronc/apprise":
-      versioning-strategy: "increase"
-      schedule:
-        interval: "daily"
     groups:
       all:
         patterns:
         - "*"
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: "daily"
+    allow:
+      - dependency-name: "apprise"

.github/workflows/containers.yml

@@ -95,7 +95,7 @@ jobs:
           push: true
           tags: |
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:dev,ghcr.io/${{ github.repository }}:dev
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
@@ -133,7 +133,7 @@ jobs:
           file: ./Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max
 # Looks like this was disabled

.github/workflows/test-container-build.yml

@@ -38,8 +38,6 @@ jobs:
             dockerfile: ./Dockerfile
           - platform: linux/arm/v8
             dockerfile: ./Dockerfile
-          - platform: linux/arm64/v8
-            dockerfile: ./Dockerfile
           # Alpine Dockerfile platforms (musl via alpine check)
           - platform: linux/amd64
             dockerfile: ./.github/test/Dockerfile-alpine

changedetectionio/processors/text_json_diff/processor.py

@@ -153,12 +153,26 @@ class perform_site_check(difference_detection_processor):
             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
             self.fetcher.content = html_tools.workarounds_for_obfuscations(self.fetcher.content)
             html_content = self.fetcher.content
+            content_type = self.fetcher.get_all_headers().get('content-type', '').lower()
+            is_attachment = 'attachment' in self.fetcher.get_all_headers().get('content-disposition', '').lower()
 
-            # If not JSON,  and if it's not text/plain..
-            if 'text/plain' in self.fetcher.get_all_headers().get('content-type', '').lower():
+            # Try to detect better mime types if its a download or not announced as HTML
+            if is_attachment or 'octet-stream' in content_type or not 'html' in content_type:
+                logger.debug(f"Got a reply that may be a download or possibly a text attachment, checking..")
+                try:
+                    import magic
+                    mime = magic.from_buffer(html_content, mime=True)
+                    logger.debug(f"Guessing mime type, original content_type '{content_type}', mime type detected '{mime}'")
+                    if mime and "/" in mime: # looks valid and is a valid mime type
+                        content_type = mime
+                except Exception as e:
+                    logger.error(f"Error getting a more precise mime type from 'magic' library ({str(e)}")
+
+            if 'text/' in content_type and not 'html' in content_type:
                 # Don't run get_text or xpath/css filters on plaintext
                 stripped_text_from_html = html_content
             else:
+                # If not JSON, and if it's not text/plain..
                 # Does it have some ld+json price data? used for easier monitoring
                 update_obj['has_ldjson_price_data'] = html_tools.has_ldjson_product_info(self.fetcher.content)
 

changedetectionio/run_socks_proxy_tests.sh

@@ -9,7 +9,7 @@ set -x
 # SOCKS5 related - start simple Socks5 proxy server
 # SOCKSTEST=xyz should show in the logs of this service to confirm it fetched
 docker run --network changedet-network -d --hostname socks5proxy --rm  --name socks5proxy -p 1080:1080 -e PROXY_USER=proxy_user123 -e PROXY_PASSWORD=proxy_pass123 serjs/go-socks5-proxy
-docker run --network changedet-network -d --hostname socks5proxy-noauth --rm  -p 1081:1080 --name socks5proxy-noauth  serjs/go-socks5-proxy
+docker run --network changedet-network -d --hostname socks5proxy-noauth --rm -p 1081:1080 --name socks5proxy-noauth -e REQUIRE_AUTH=false serjs/go-socks5-proxy
 
 echo "---------------------------------- SOCKS5 -------------------"
 # SOCKS5 related - test from proxies.json

changedetectionio/tests/test_backend.py

@@ -165,3 +165,53 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # Cleanup everything
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
+
+def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage):
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("""some random text that should be split by line
+and not parsed with html_to_text
+this way we know that it correctly parsed as plain text
+\r\n
+ok\r\n
+got it\r\n
+""")
+
+    test_url = url_for('test_endpoint', content_type="application/octet-stream", _external=True)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("imports.import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    wait_for_all_checks(client)
+
+    ### check the front end
+    res = client.get(
+        url_for("ui.ui_views.preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"some random text that should be split by line\n" in res.data
+    ####
+
+    # Check the snapshot by API that it has linefeeds too
+    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+    res = client.get(
+        url_for("watchhistory", uuid=watch_uuid),
+        headers={'x-api-key': api_key},
+    )
+
+    # Fetch a snapshot by timestamp, check the right one was found
+    res = client.get(
+        url_for("watchsinglehistory", uuid=watch_uuid, timestamp=list(res.json.keys())[-1]),
+        headers={'x-api-key': api_key},
+    )
+    assert b"some random text that should be split by line\n" in res.data
+
+
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
+

changedetectionio/tests/util.py

@@ -130,40 +130,47 @@ def extract_UUID_from_client(client):
 
 def wait_for_all_checks(client=None):
     """
-    Waits until the queue is empty and workers are idle.
-    Much faster than the original with adaptive timing.
+    Waits until both queues are empty and all workers are idle.
+    Optimized for Janus queues with minimal delays.
     """
-    from changedetectionio.flask_app import update_q as global_update_q
+    from changedetectionio.flask_app import update_q as global_update_q, notification_q
     from changedetectionio import worker_handler
 
     logger = logging.getLogger()
-    empty_since = None
-    attempt = 0
-    max_attempts = 150  # Still reasonable upper bound
 
-    while attempt < max_attempts:
-        # Start with fast checks, slow down if needed
-        if attempt < 10:
-            time.sleep(0.1)  # Very fast initial checks
-        elif attempt < 30:
-            time.sleep(0.3)  # Medium speed
-        else:
-            time.sleep(0.8)  # Slower for persistent issues
+    # Much tighter timing with reliable Janus queues
+    max_attempts = 100  # Reduced from 150
 
-        q_length = global_update_q.qsize()
+    for attempt in range(max_attempts):
+        # Check both queues and worker status
+        update_q_size = global_update_q.qsize()
+        notification_q_size = notification_q.qsize()
         running_uuids = worker_handler.get_running_uuids()
         any_workers_busy = len(running_uuids) > 0
 
-        if q_length == 0 and not any_workers_busy:
-            if empty_since is None:
-                empty_since = time.time()
-            elif time.time() - empty_since >= 0.15:  # Shorter wait
-                break
+        # Both queues empty and no workers processing
+        if update_q_size == 0 and notification_q_size == 0 and not any_workers_busy:
+            # Small delay to account for items being added to queue during processing
+            time.sleep(0.05)
+
+            # Double-check after brief delay
+            update_q_size = global_update_q.qsize()
+            notification_q_size = notification_q.qsize()
+            running_uuids = worker_handler.get_running_uuids()
+            any_workers_busy = len(running_uuids) > 0
+
+            if update_q_size == 0 and notification_q_size == 0 and not any_workers_busy:
+                return  # All clear!
+
+        # Adaptive sleep timing - start fast, get slightly slower
+        if attempt < 20:
+            time.sleep(0.05)  # Very fast initial checks
+        elif attempt < 50:
+            time.sleep(0.1)   # Medium speed
         else:
-            empty_since = None
-        
-        attempt += 1
-        time.sleep(0.3)
+            time.sleep(0.2)   # Slower for edge cases
+
+    logger.warning(f"wait_for_all_checks() timed out after {max_attempts} attempts")
 
 # Replaced by new_live_server_setup and calling per function scope in conftest.py
 def  live_server_setup(live_server):

requirements.txt

@@ -39,7 +39,7 @@ jsonpath-ng~=1.5.3
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise==1.9.3
+apprise==1.9.4
 
 # - Needed for apprise/spush, and maybe others? hopefully doesnt trigger a rust compile.
 # - Requires extra wheel for rPi, adds build time for arm/v8 which is not in piwheels
@@ -51,8 +51,8 @@ cryptography==44.0.1
 # use any version other than 2.0.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
 paho-mqtt!=2.0.*
 
-# Used for CSS filtering
-beautifulsoup4>=4.0.0
+# Used for CSS filtering, JSON extraction from HTML
+beautifulsoup4>=4.0.0,<=4.13.5
 
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
 # #2328 - 5.2.0 and 5.2.1 had extra CPU flag CFLAGS set which was not compatible on older hardware