WIP

changedetectionio/__init__.py

@@ -10,7 +10,6 @@ from json.decoder import JSONDecodeError
 from loguru import logger
 import getopt
 import logging
-import os
 import platform
 import signal
 import threading

changedetectionio/content_fetchers/__init__.py

@@ -1,5 +1,4 @@
 import sys
-from changedetectionio.strtobool import strtobool
 from loguru import logger
 from changedetectionio.content_fetchers.exceptions import BrowserStepsStepException
 import os
@@ -76,19 +75,6 @@ def _load_fetchers():
         logger.error(f"Error loading plugin fetchers: {e}")
 
 
-def get_active_browser_fetcher_name() -> str:
-    """Return the clean name of the browser fetcher activated by environment config.
-
-    - ``PLAYWRIGHT_DRIVER_URL`` set + ``FAST_PUPPETEER_CHROME_FETCHER=False`` → ``playwright_cdp``
-    - ``PLAYWRIGHT_DRIVER_URL`` set + ``FAST_PUPPETEER_CHROME_FETCHER=True``  → ``puppeteer``
-    - Neither set → ``selenium``
-    """
-    if os.getenv('PLAYWRIGHT_DRIVER_URL', False):
-        if not strtobool(os.getenv('FAST_PUPPETEER_CHROME_FETCHER', 'False')):
-            return 'playwright_cdp'
-        return 'puppeteer'
-    return 'selenium'
-
 
 # Default browser profiles always shown in the browser profiles table (keyed by machine name)
 DEFAULT_BROWSER_PROFILES: dict = {}

changedetectionio/content_fetchers/base.py

@@ -88,6 +88,8 @@ class Fetcher():
     profile_user_agent: str = None   # Profile-level UA; lower priority than request_headers User-Agent
     ignore_https_errors: bool = False
     locale: str = None
+    service_workers: str = 'allow'
+    extra_delay: int = 0
 
     def __init__(self, **kwargs):
         if kwargs and 'screenshot_format' in kwargs:
@@ -98,7 +100,8 @@ class Fetcher():
 
         # BrowserProfile fields — store whatever was passed, subclasses use them
         for field in ('viewport_width', 'viewport_height', 'block_images', 'block_fonts',
-                      'profile_user_agent', 'ignore_https_errors', 'locale'):
+                      'profile_user_agent', 'ignore_https_errors', 'locale',
+                      'service_workers', 'extra_delay'):
             if field in kwargs:
                 setattr(self, field, kwargs[field])
 

changedetectionio/content_fetchers/playwright/CDP.py

@@ -6,8 +6,6 @@ method explicit. The PLAYWRIGHT_DRIVER_URL env var (or per-profile
 browser_connection_url) points to a running Chrome/Chromium container that
 exposes the CDP WebSocket endpoint (e.g. ws://playwright-chrome:3000).
 """
-import os
-
 from changedetectionio.pluggy_interface import hookimpl
 from changedetectionio.content_fetchers.playwright import PlaywrightBaseFetcher
 
@@ -22,10 +20,10 @@ class fetcher(PlaywrightBaseFetcher):
             self.browser_connection_is_custom = True
             self.browser_connection_url = custom_browser_connection_url
         else:
-            self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
+            self.browser_connection_url = 'ws://playwright-chrome:3000'
 
-        # CDP always talks to Chromium; respect PLAYWRIGHT_BROWSER_TYPE for exotic setups
-        self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
+        # CDP always connects to Chromium
+        self.browser_type = 'chromium'
 
     async def _connect_browser(self, p):
         browser_type = getattr(p, self.browser_type)

changedetectionio/content_fetchers/playwright/__init__.py

@@ -261,7 +261,7 @@ class PlaywrightBaseFetcher(Fetcher):
                 extra_http_headers=request_headers,
                 ignore_https_errors=self.ignore_https_errors,
                 proxy=self.proxy,
-                service_workers=os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow'),
+                service_workers=self.service_workers,
                 user_agent=ua,
                 viewport={'width': self.viewport_width, 'height': self.viewport_height},
             )
@@ -312,7 +312,7 @@ class PlaywrightBaseFetcher(Fetcher):
                 await browser.close()
                 raise PageUnloadable(url=url, status_code=None, message=str(e))
 
-            extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
+            extra_wait = self.extra_delay + self.render_extract_delay
             await self.page.wait_for_timeout(extra_wait * 1000)
 
             try:

changedetectionio/content_fetchers/puppeteer.py

@@ -191,9 +191,7 @@ class fetcher(Fetcher):
             self.browser_connection_is_custom = True
             self.browser_connection_url = custom_browser_connection_url
         else:
-            # Fallback to fetching from system
-            # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-            self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
+            self.browser_connection_url = 'ws://playwright-chrome:3000'
 
         # allow per-watch proxy selection override
         # @todo check global too?
@@ -263,7 +261,7 @@ class fetcher(Fetcher):
         import re
         self.delete_browser_steps_screenshots()
 
-        n = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 12)) + self.render_extract_delay
+        n = self.extra_delay + self.render_extract_delay
         extra_wait = min(n, 15)
 
         logger.debug(f"Extra wait set to {extra_wait}s, requested was {n}s.")

changedetectionio/content_fetchers/webdriver_selenium.py

@@ -24,12 +24,11 @@ class fetcher(Fetcher):
         from urllib.parse import urlparse
         from selenium.webdriver.common.proxy import Proxy
 
-        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-        if not custom_browser_connection_url:
-            self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
-        else:
+        if custom_browser_connection_url:
             self.browser_connection_is_custom = True
             self.browser_connection_url = custom_browser_connection_url
+        else:
+            self.browser_connection_url = 'http://browser-chrome:4444/wd/hub'
 
         ##### PROXY SETUP #####
 
@@ -121,12 +120,12 @@ class fetcher(Fetcher):
                 if not "--window-size" in os.getenv("CHROME_OPTIONS", ""):
                     driver.set_window_size(1280, 1024)
 
-                driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+                driver.implicitly_wait(self.extra_delay)
 
                 if self.webdriver_js_execute_code is not None:
                     driver.execute_script(self.webdriver_js_execute_code)
                     # Selenium doesn't automatically wait for actions as good as Playwright, so wait again
-                    driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+                    driver.implicitly_wait(self.extra_delay)
 
                 # @todo - how to check this? is it possible?
                 self.status_code = 200
@@ -135,7 +134,7 @@ class fetcher(Fetcher):
 
                 # @todo - dom wait loaded?
                 import time
-                time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay)
+                time.sleep(self.extra_delay + self.render_extract_delay)
                 self.content = driver.page_source
                 self.headers = {}
 

changedetectionio/model/browser_profile.py

@@ -139,6 +139,20 @@ class BrowserProfile(BaseModel):
     Some sites serve different prices or copy based on locale.
     """
 
+    service_workers: str = 'allow'
+    """
+    Whether to allow Service Workers in the browser context.
+    Playwright accepts ``'allow'`` or ``'block'``.
+    Block to avoid large Service Worker data transfers (e.g. YouTube).
+    """
+
+    extra_delay: int = 0
+    """
+    Extra seconds to wait after page load before extracting content
+    (on top of the per-watch ``render_extract_delay``).
+    Sourced from ``WEBDRIVER_DELAY_BEFORE_CONTENT_READY`` at startup.
+    """
+
     model_config = {"frozen": False}
 
     # ------------------------------------------------------------------
@@ -261,21 +275,19 @@ RESERVED_MACHINE_NAMES: frozenset[str] = frozenset(_BUILTINS.keys())
 
 
 def get_default_browser_builtin() -> BrowserProfile:
-    """Return the built-in browser profile that matches the current environment.
+    """Return the built-in browser profile configured by the environment.
 
-    Reads the same env vars as ``content_fetchers.get_active_browser_fetcher_name()``:
-
-    * ``PLAYWRIGHT_DRIVER_URL`` set + ``FAST_PUPPETEER_CHROME_FETCHER=False`` → Playwright
-    * ``PLAYWRIGHT_DRIVER_URL`` set + ``FAST_PUPPETEER_CHROME_FETCHER=True``  → Puppeteer
-    * Neither set → Selenium
+    ``preconfigure_browsers_based_on_env()`` sets ``browser_connection_url`` on
+    the relevant built-in at startup.  We just check which one has a URL.
+    Falls back to ``BUILTIN_SELENIUM`` when nothing is configured.
     """
-    import os
-    from changedetectionio.strtobool import strtobool
-    if os.getenv('PLAYWRIGHT_DRIVER_URL', False):
-        if not strtobool(os.getenv('FAST_PUPPETEER_CHROME_FETCHER', 'False')):
-            return BUILTIN_PLAYWRIGHT
+    if BUILTIN_PLAYWRIGHT.browser_connection_url:
+        return BUILTIN_PLAYWRIGHT
+    if BUILTIN_PUPPETEER.browser_connection_url:
         return BUILTIN_PUPPETEER
-    return BUILTIN_SELENIUM
+    if BUILTIN_SELENIUM.browser_connection_url:
+        return BUILTIN_SELENIUM
+    return BUILTIN_REQUESTS
 
 
 # ---------------------------------------------------------------------------
@@ -351,4 +363,4 @@ def resolve_browser_profile(watch, datastore) -> BrowserProfile:
             f"falling back through the chain"
         )
 
-    return BUILTIN_REQUESTS
+    return get_default_browser_builtin()

changedetectionio/processors/base.py

@@ -177,6 +177,8 @@ class difference_detection_processor():
             profile_user_agent=profile.user_agent,
             ignore_https_errors=profile.ignore_https_errors,
             locale=profile.locale,
+            service_workers=profile.service_workers,
+            extra_delay=profile.extra_delay,
         )
 
         if self.watch.has_browser_steps:

changedetectionio/store/__init__.py

@@ -218,6 +218,8 @@ class ChangeDetectionStore(DatastoreUpdatesMixin, FileSavingDataStore):
         changedetection_json = os.path.join(self.datastore_path, "changedetection.json")
         changedetection_json_old_schema = os.path.join(self.datastore_path, "url-watches.json")
 
+        self.preconfigure_browsers_based_on_env()
+
         if os.path.exists(changedetection_json):
             # Run schema updates if needed
             # Pass current schema version from loaded datastore (defaults to 0 if not set)
@@ -334,6 +336,38 @@ class ChangeDetectionStore(DatastoreUpdatesMixin, FileSavingDataStore):
         entity = watch_class(datastore_path=self.datastore_path, __datastore=self, default=entity)
         return entity
 
+    def preconfigure_browsers_based_on_env(self):
+        """Set browser_connection_url on built-in profiles from environment variables.
+
+        Called once at datastore init (before _load_state).  Mutates the module-level
+        _BUILTINS singletons so the URLs are visible to every profile lookup for the
+        lifetime of the process without needing to touch persisted settings.
+        """
+        from changedetectionio.model import browser_profile as bp
+        from changedetectionio.strtobool import strtobool
+
+        playwright_url = os.getenv('PLAYWRIGHT_DRIVER_URL')
+        if playwright_url:
+            playwright_url = playwright_url.strip('"')
+            if strtobool(os.getenv('FAST_PUPPETEER_CHROME_FETCHER', 'False')):
+                bp.BUILTIN_PUPPETEER.browser_connection_url = playwright_url
+            else:
+                bp.BUILTIN_PLAYWRIGHT.browser_connection_url = playwright_url
+
+        webdriver_url = os.getenv('WEBDRIVER_URL')
+        if webdriver_url:
+            bp.BUILTIN_SELENIUM.browser_connection_url = webdriver_url.strip('"')
+
+        service_workers = os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow')
+        bp.BUILTIN_PLAYWRIGHT.service_workers = service_workers
+        bp.BUILTIN_PUPPETEER.service_workers = service_workers
+
+        extra_delay = int(os.getenv('WEBDRIVER_DELAY_BEFORE_CONTENT_READY', 0))
+        bp.BUILTIN_PLAYWRIGHT.extra_delay = extra_delay
+        bp.BUILTIN_PUPPETEER.extra_delay = extra_delay
+        bp.BUILTIN_SELENIUM.extra_delay = extra_delay
+
+
     # ============================================================================
     # FileSavingDataStore Abstract Method Implementations
     # ============================================================================

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -22,7 +22,7 @@ def test_preferred_proxy(client, live_server, measure_memory_usage, datastore_pa
         url_for("ui.ui_edit.edit_page", uuid="first", unpause_on_save=1),
         data={
                 "include_filters": "",
-                "browser_profile": 'browser_chromeplaywright' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'direct_http_requests',
+                "browser_profile": "system",
                 "headers": "",
                 "proxy": "proxy-two",
                 "tags": "",

changedetectionio/tests/proxy_list/test_proxy_noconnect.py

@@ -41,7 +41,7 @@ def test_proxy_noconnect_custom(client, live_server, measure_memory_usage, datas
 
     options = {
         "url": test_url,
-        "browser_profile": "browser_chromeplaywright" if os.getenv('PLAYWRIGHT_DRIVER_URL') or os.getenv("WEBDRIVER_URL") else "direct_http_requests",
+        "browser_profile": "system",
         "proxy": "ui-0custom-test-proxy",
         "time_between_check_use_default": "y",
     }

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -60,7 +60,7 @@ def test_socks5(client, live_server, measure_memory_usage, datastore_path):
         url_for("ui.ui_edit.edit_page", uuid="first", unpause_on_save=1),
         data={
             "include_filters": "",
-            "browser_profile": 'browser_chromeplaywright' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'direct_http_requests',
+            "browser_profile": "system",
             "headers": "",
             "proxy": "ui-0socks5proxy",
             "tags": "",

changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py

@@ -48,7 +48,7 @@ def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage,
         url_for("ui.ui_edit.edit_page", uuid="first", unpause_on_save=1),
         data={
             "include_filters": "",
-            "browser_profile": 'browser_chromeplaywright' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'direct_http_requests',
+            "browser_profile": "system",
             "headers": "",
             "proxy": "socks5proxy",
             "tags": "",

changedetectionio/tests/test_request.py

@@ -35,7 +35,7 @@ def test_headers_in_request(client, live_server, measure_memory_usage, datastore
         data={
               "url": test_url,
               "tags": "",
-              "browser_profile": 'browser_chromeplaywright' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'direct_http_requests',
+              "browser_profile": "system",
               "headers": "jinja2:{{ 1+1 }}\nxxx:ooo\ncool:yeah\r\ncookie:"+cookie_header,
               "time_between_check_use_default": "y"},
         follow_redirects=True
@@ -345,7 +345,7 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage,
         data={
             "url": test_url,
             "tags": "testtag",
-            "browser_profile": 'browser_chromeplaywright' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'direct_http_requests',
+            "browser_profile": "system",
             "headers": "xxx:ooo\ncool:yeah\r\n",
             "time_between_check_use_default": "y"},
         follow_redirects=True