Merge branch 'master' into python312

.github/workflows/containers.yml

@@ -40,10 +40,10 @@ jobs:
     if: ${{ github.event.workflow_run.conclusion == 'success' }} || ${{ github.event.release.tag_name }} != ''
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: 3.12
 
       - name: Install dependencies
         run: |

.github/workflows/pypi-release.yml

@@ -11,7 +11,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.12"
     - name: Install pypa/build
       run: >-
         python3 -m
@@ -38,10 +38,10 @@ jobs:
       with:
         name: python-package-distributions
         path: dist/
-    - name: Set up Python 3.11
+    - name: Set up Python 3.12
       uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.12'
     - name: Test that the basic pip built package runs without error
       run: |
         set -ex

.github/workflows/test-container-build.yml

@@ -27,10 +27,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
         - uses: actions/checkout@v4
-        - name: Set up Python 3.11
+        - name: Set up Python 3.12
           uses: actions/setup-python@v5
           with:
-            python-version: 3.11
+            python-version: 3.12
 
         # Just test that the build works, some libraries won't compile on ARM/rPi etc
         - name: Set up QEMU

.github/workflows/test-only.yml

@@ -10,10 +10,10 @@ jobs:
       - uses: actions/checkout@v4
 
       # Mainly just for link/flake8
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.12'
 
       - name: Lint with flake8
         run: |

Dockerfile

@@ -2,7 +2,7 @@
 
 # @NOTE! I would love to move to 3.11 but it breaks the async handler in changedetectionio/content_fetchers/puppeteer.py
 #        If you know how to fix it, please do! and test it for both 3.10 and 3.11
-FROM python:3.10-slim-bookworm as builder
+FROM python:3.12-slim-bookworm as builder
 
 # See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -32,7 +32,7 @@ RUN pip install --target=/dependencies playwright~=1.41.2 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.10-slim-bookworm
+FROM python:3.12-slim-bookworm
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libxslt1.1 \

changedetectionio/content_fetchers/base.py

@@ -28,7 +28,7 @@ def manage_user_agent(headers, current_ua=''):
     :return:
     """
     # Ask it what the user agent is, if its obviously ChromeHeadless, switch it to the default
-    ua_in_custom_headers = headers.get('User-Agent')
+    ua_in_custom_headers = next((v for k, v in headers.items() if k.lower() == "user-agent"), None)
     if ua_in_custom_headers:
         return ua_in_custom_headers
 

changedetectionio/content_fetchers/puppeteer.py

@@ -115,11 +115,12 @@ class fetcher(Fetcher):
 
         # This user agent is similar to what was used when tweaking the evasions in inject_evasions_into_page(..)
         user_agent = None
-        if request_headers and request_headers.get('User-Agent'):
-            # Request_headers should now be CaaseInsensitiveDict
-            # Remove it so it's not sent again with headers after
-            user_agent = request_headers.pop('User-Agent').strip()
-            await self.page.setUserAgent(user_agent)
+        if request_headers:
+            user_agent = next((value for key, value in request_headers.items() if key.lower().strip() == 'user-agent'), None)
+            if user_agent:
+                await self.page.setUserAgent(user_agent)
+                # Remove it so it's not sent again with headers after
+                [request_headers.pop(key) for key in list(request_headers) if key.lower().strip() == 'user-agent'.lower().strip()]
 
         if not user_agent:
             # Attempt to strip 'HeadlessChrome' etc

changedetectionio/model/Watch.py

@@ -1,6 +1,5 @@
 from changedetectionio.strtobool import strtobool
 from changedetectionio.safe_jinja import render as jinja_render
-
 import os
 import re
 import time
@@ -333,9 +332,7 @@ class model(dict):
         # Small hack so that we sleep just enough to allow 1 second  between history snapshots
         # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
         if self.__newest_history_key and int(timestamp) == int(self.__newest_history_key):
-            logger.warning(f"Timestamp {timestamp} already exists, waiting 1 seconds so we have a unique key in history.txt")
-            timestamp = str(int(timestamp) + 1)
-            time.sleep(1)
+            time.sleep(timestamp - self.__newest_history_key)
 
         threshold = int(os.getenv('SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD', 1024))
         skip_brotli = strtobool(os.getenv('DISABLE_BROTLI_TEXT_SNAPSHOT', 'False'))

changedetectionio/processors/__init__.py

@@ -1,10 +1,10 @@
 from abc import abstractmethod
-from changedetectionio.strtobool import strtobool
-from copy import deepcopy
-from loguru import logger
-import hashlib
 import os
+import hashlib
 import re
+from copy import deepcopy
+from changedetectionio.strtobool import strtobool
+from loguru import logger
 
 class difference_detection_processor():
 
@@ -21,7 +21,7 @@ class difference_detection_processor():
         self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
 
     def call_browser(self):
-        from requests.structures import CaseInsensitiveDict
+
         # Protect against file:// access
         if re.search(r'^file://', self.watch.get('url', '').strip(), re.IGNORECASE):
             if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
@@ -93,16 +93,14 @@ class difference_detection_processor():
             self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid'))
 
         # Tweak the base config with the per-watch ones
-        request_headers = CaseInsensitiveDict()
+        request_headers = self.watch.get('headers', [])
+        request_headers.update(self.datastore.get_all_base_headers())
+        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=self.watch.get('uuid')))
 
         ua = self.datastore.data['settings']['requests'].get('default_ua')
         if ua and ua.get(prefer_fetch_backend):
             request_headers.update({'User-Agent': ua.get(prefer_fetch_backend)})
 
-        request_headers.update(self.watch.get('headers', {}))
-        request_headers.update(self.datastore.get_all_base_headers())
-        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=self.watch.get('uuid')))
-
         # https://github.com/psf/requests/issues/4525
         # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
         # do this by accident.

changedetectionio/store.py

@@ -178,7 +178,7 @@ class ChangeDetectionStore:
     @property
     def has_unviewed(self):
         for uuid, watch in self.__data['watching'].items():
-            if watch.history_n >= 2 and watch.viewed == False:
+            if watch.viewed == False:
                 return True
         return False
 

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -71,8 +71,6 @@ def test_check_notification_email_formats_default_HTML(client, live_server):
 
     wait_for_all_checks(client)
     set_longer_modified_response()
-    time.sleep(2)
-
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -137,7 +135,6 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
     wait_for_all_checks(client)
     set_longer_modified_response()
-    time.sleep(2)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -105,7 +105,6 @@ def test_check_ldjson_price_autodetect(client, live_server):
     wait_for_all_checks(client)
 
     # Trigger a check
-    time.sleep(1)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     # Offer should be gone

changedetectionio/tests/test_backend.py

@@ -135,9 +135,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
     # It should have picked up the <title>
     assert b'head title' in res.data
 
-    # Be sure the last_viewed is going to be greater than the last snapshot
-    time.sleep(1)
-
     # hit the mark all viewed link
     res = client.get(url_for("mark_all_viewed"), follow_redirects=True)
 

changedetectionio/tests/test_notification_errors.py

@@ -9,6 +9,9 @@ def test_check_notification_error_handling(client, live_server):
     live_server_setup(live_server)
     set_original_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Set a URL and fetch it, then set a notification URL which is going to give errors
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(

changedetectionio/tests/test_request.py

@@ -253,62 +253,6 @@ def test_method_in_request(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-# Re #2408 - user-agent override test, also should handle case-insensitive header deduplication
-def test_ua_global_override(client, live_server):
-    # live_server_setup(live_server)
-    test_url = url_for('test_headers', _external=True)
-
-    res = client.post(
-        url_for("settings_page"),
-        data={
-            "application-fetch_backend": "html_requests",
-            "application-minutes_between_check": 180,
-            "requests-default_ua-html_requests": "html-requests-user-agent"
-        },
-        follow_redirects=True
-    )
-    assert b'Settings updated' in res.data
-
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    wait_for_all_checks(client)
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert b"html-requests-user-agent" in res.data
-    # default user-agent should have shown by now
-    # now add a custom one in the headers
-
-
-    # Add some headers to a request
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-            "url": test_url,
-            "tags": "testtag",
-            "fetch_backend": 'html_requests',
-            # Important - also test case-insensitive
-            "headers": "User-AGent: agent-from-watch"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    assert b"agent-from-watch" in res.data
-    assert b"html-requests-user-agent" not in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
 def test_headers_textfile_in_request(client, live_server):
     #live_server_setup(live_server)
     # Add our URL to the import page
@@ -389,7 +333,7 @@ def test_headers_textfile_in_request(client, live_server):
     # Not needed anymore
     os.unlink('test-datastore/headers.txt')
     os.unlink('test-datastore/headers-testtag.txt')
-
+    os.unlink('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt')
     # The service should echo back the request verb
     res = client.get(
         url_for("preview_page", uuid="first"),

requirements.txt

@@ -1,6 +1,7 @@
 # Used by Pyppeteer
 pyee
-
+# eventlet 0.33.3 was related to dnspython fixes
+# 0.34.1 - fixes python 3.12 "AttributeError: module 'ssl' has no attribute 'wrap_socket'"
 eventlet==0.35.2 # related to dnspython fixes
 feedgen~=0.9
 flask-compress