From 93e48eff230282ee886fc5e546fae08ab708a6d5 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 26 Mar 2026 15:16:30 +0100 Subject: [PATCH] WIP --- changedetectionio/__init__.py | 1 - .../content_fetchers/__init__.py | 14 ------- changedetectionio/content_fetchers/base.py | 5 ++- .../content_fetchers/playwright/CDP.py | 8 ++-- .../content_fetchers/playwright/__init__.py | 4 +- .../content_fetchers/puppeteer.py | 6 +-- .../content_fetchers/webdriver_selenium.py | 13 +++---- changedetectionio/model/browser_profile.py | 38 ++++++++++++------- changedetectionio/processors/base.py | 2 + changedetectionio/store/__init__.py | 34 +++++++++++++++++ .../tests/proxy_list/test_multiple_proxy.py | 2 +- .../tests/proxy_list/test_proxy_noconnect.py | 2 +- .../tests/proxy_socks5/test_socks5_proxy.py | 2 +- .../proxy_socks5/test_socks5_proxy_sources.py | 2 +- changedetectionio/tests/test_request.py | 4 +- 15 files changed, 84 insertions(+), 53 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 8a0cc8862..742c05346 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -10,7 +10,6 @@ from json.decoder import JSONDecodeError from loguru import logger import getopt import logging -import os import platform import signal import threading diff --git a/changedetectionio/content_fetchers/__init__.py b/changedetectionio/content_fetchers/__init__.py index 839cd8a21..da2164e31 100644 --- a/changedetectionio/content_fetchers/__init__.py +++ b/changedetectionio/content_fetchers/__init__.py @@ -1,5 +1,4 @@ import sys -from changedetectionio.strtobool import strtobool from loguru import logger from changedetectionio.content_fetchers.exceptions import BrowserStepsStepException import os @@ -76,19 +75,6 @@ def _load_fetchers(): logger.error(f"Error loading plugin fetchers: {e}") -def get_active_browser_fetcher_name() -> str: - """Return the clean name of the browser fetcher activated by environment config. - - - ``PLAYWRIGHT_DRIVER_URL`` set + ``FAST_PUPPETEER_CHROME_FETCHER=False`` → ``playwright_cdp`` - - ``PLAYWRIGHT_DRIVER_URL`` set + ``FAST_PUPPETEER_CHROME_FETCHER=True`` → ``puppeteer`` - - Neither set → ``selenium`` - """ - if os.getenv('PLAYWRIGHT_DRIVER_URL', False): - if not strtobool(os.getenv('FAST_PUPPETEER_CHROME_FETCHER', 'False')): - return 'playwright_cdp' - return 'puppeteer' - return 'selenium' - # Default browser profiles always shown in the browser profiles table (keyed by machine name) DEFAULT_BROWSER_PROFILES: dict = {} diff --git a/changedetectionio/content_fetchers/base.py b/changedetectionio/content_fetchers/base.py index 6631afe88..35786542f 100644 --- a/changedetectionio/content_fetchers/base.py +++ b/changedetectionio/content_fetchers/base.py @@ -88,6 +88,8 @@ class Fetcher(): profile_user_agent: str = None # Profile-level UA; lower priority than request_headers User-Agent ignore_https_errors: bool = False locale: str = None + service_workers: str = 'allow' + extra_delay: int = 0 def __init__(self, **kwargs): if kwargs and 'screenshot_format' in kwargs: @@ -98,7 +100,8 @@ class Fetcher(): # BrowserProfile fields — store whatever was passed, subclasses use them for field in ('viewport_width', 'viewport_height', 'block_images', 'block_fonts', - 'profile_user_agent', 'ignore_https_errors', 'locale'): + 'profile_user_agent', 'ignore_https_errors', 'locale', + 'service_workers', 'extra_delay'): if field in kwargs: setattr(self, field, kwargs[field]) diff --git a/changedetectionio/content_fetchers/playwright/CDP.py b/changedetectionio/content_fetchers/playwright/CDP.py index 7a2f08a49..168f190ad 100644 --- a/changedetectionio/content_fetchers/playwright/CDP.py +++ b/changedetectionio/content_fetchers/playwright/CDP.py @@ -6,8 +6,6 @@ method explicit. The PLAYWRIGHT_DRIVER_URL env var (or per-profile browser_connection_url) points to a running Chrome/Chromium container that exposes the CDP WebSocket endpoint (e.g. ws://playwright-chrome:3000). """ -import os - from changedetectionio.pluggy_interface import hookimpl from changedetectionio.content_fetchers.playwright import PlaywrightBaseFetcher @@ -22,10 +20,10 @@ class fetcher(PlaywrightBaseFetcher): self.browser_connection_is_custom = True self.browser_connection_url = custom_browser_connection_url else: - self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"') + self.browser_connection_url = 'ws://playwright-chrome:3000' - # CDP always talks to Chromium; respect PLAYWRIGHT_BROWSER_TYPE for exotic setups - self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"') + # CDP always connects to Chromium + self.browser_type = 'chromium' async def _connect_browser(self, p): browser_type = getattr(p, self.browser_type) diff --git a/changedetectionio/content_fetchers/playwright/__init__.py b/changedetectionio/content_fetchers/playwright/__init__.py index 74f119d08..c4da927c0 100644 --- a/changedetectionio/content_fetchers/playwright/__init__.py +++ b/changedetectionio/content_fetchers/playwright/__init__.py @@ -261,7 +261,7 @@ class PlaywrightBaseFetcher(Fetcher): extra_http_headers=request_headers, ignore_https_errors=self.ignore_https_errors, proxy=self.proxy, - service_workers=os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow'), + service_workers=self.service_workers, user_agent=ua, viewport={'width': self.viewport_width, 'height': self.viewport_height}, ) @@ -312,7 +312,7 @@ class PlaywrightBaseFetcher(Fetcher): await browser.close() raise PageUnloadable(url=url, status_code=None, message=str(e)) - extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay + extra_wait = self.extra_delay + self.render_extract_delay await self.page.wait_for_timeout(extra_wait * 1000) try: diff --git a/changedetectionio/content_fetchers/puppeteer.py b/changedetectionio/content_fetchers/puppeteer.py index 95df9b557..0b1a19010 100644 --- a/changedetectionio/content_fetchers/puppeteer.py +++ b/changedetectionio/content_fetchers/puppeteer.py @@ -191,9 +191,7 @@ class fetcher(Fetcher): self.browser_connection_is_custom = True self.browser_connection_url = custom_browser_connection_url else: - # Fallback to fetching from system - # .strip('"') is going to save someone a lot of time when they accidently wrap the env value - self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"') + self.browser_connection_url = 'ws://playwright-chrome:3000' # allow per-watch proxy selection override # @todo check global too? @@ -263,7 +261,7 @@ class fetcher(Fetcher): import re self.delete_browser_steps_screenshots() - n = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 12)) + self.render_extract_delay + n = self.extra_delay + self.render_extract_delay extra_wait = min(n, 15) logger.debug(f"Extra wait set to {extra_wait}s, requested was {n}s.") diff --git a/changedetectionio/content_fetchers/webdriver_selenium.py b/changedetectionio/content_fetchers/webdriver_selenium.py index 4434ecb29..48e5d368c 100644 --- a/changedetectionio/content_fetchers/webdriver_selenium.py +++ b/changedetectionio/content_fetchers/webdriver_selenium.py @@ -24,12 +24,11 @@ class fetcher(Fetcher): from urllib.parse import urlparse from selenium.webdriver.common.proxy import Proxy - # .strip('"') is going to save someone a lot of time when they accidently wrap the env value - if not custom_browser_connection_url: - self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"') - else: + if custom_browser_connection_url: self.browser_connection_is_custom = True self.browser_connection_url = custom_browser_connection_url + else: + self.browser_connection_url = 'http://browser-chrome:4444/wd/hub' ##### PROXY SETUP ##### @@ -121,12 +120,12 @@ class fetcher(Fetcher): if not "--window-size" in os.getenv("CHROME_OPTIONS", ""): driver.set_window_size(1280, 1024) - driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))) + driver.implicitly_wait(self.extra_delay) if self.webdriver_js_execute_code is not None: driver.execute_script(self.webdriver_js_execute_code) # Selenium doesn't automatically wait for actions as good as Playwright, so wait again - driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))) + driver.implicitly_wait(self.extra_delay) # @todo - how to check this? is it possible? self.status_code = 200 @@ -135,7 +134,7 @@ class fetcher(Fetcher): # @todo - dom wait loaded? import time - time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay) + time.sleep(self.extra_delay + self.render_extract_delay) self.content = driver.page_source self.headers = {} diff --git a/changedetectionio/model/browser_profile.py b/changedetectionio/model/browser_profile.py index f84461880..4361d86e7 100644 --- a/changedetectionio/model/browser_profile.py +++ b/changedetectionio/model/browser_profile.py @@ -139,6 +139,20 @@ class BrowserProfile(BaseModel): Some sites serve different prices or copy based on locale. """ + service_workers: str = 'allow' + """ + Whether to allow Service Workers in the browser context. + Playwright accepts ``'allow'`` or ``'block'``. + Block to avoid large Service Worker data transfers (e.g. YouTube). + """ + + extra_delay: int = 0 + """ + Extra seconds to wait after page load before extracting content + (on top of the per-watch ``render_extract_delay``). + Sourced from ``WEBDRIVER_DELAY_BEFORE_CONTENT_READY`` at startup. + """ + model_config = {"frozen": False} # ------------------------------------------------------------------ @@ -261,21 +275,19 @@ RESERVED_MACHINE_NAMES: frozenset[str] = frozenset(_BUILTINS.keys()) def get_default_browser_builtin() -> BrowserProfile: - """Return the built-in browser profile that matches the current environment. + """Return the built-in browser profile configured by the environment. - Reads the same env vars as ``content_fetchers.get_active_browser_fetcher_name()``: - - * ``PLAYWRIGHT_DRIVER_URL`` set + ``FAST_PUPPETEER_CHROME_FETCHER=False`` → Playwright - * ``PLAYWRIGHT_DRIVER_URL`` set + ``FAST_PUPPETEER_CHROME_FETCHER=True`` → Puppeteer - * Neither set → Selenium + ``preconfigure_browsers_based_on_env()`` sets ``browser_connection_url`` on + the relevant built-in at startup. We just check which one has a URL. + Falls back to ``BUILTIN_SELENIUM`` when nothing is configured. """ - import os - from changedetectionio.strtobool import strtobool - if os.getenv('PLAYWRIGHT_DRIVER_URL', False): - if not strtobool(os.getenv('FAST_PUPPETEER_CHROME_FETCHER', 'False')): - return BUILTIN_PLAYWRIGHT + if BUILTIN_PLAYWRIGHT.browser_connection_url: + return BUILTIN_PLAYWRIGHT + if BUILTIN_PUPPETEER.browser_connection_url: return BUILTIN_PUPPETEER - return BUILTIN_SELENIUM + if BUILTIN_SELENIUM.browser_connection_url: + return BUILTIN_SELENIUM + return BUILTIN_REQUESTS # --------------------------------------------------------------------------- @@ -351,4 +363,4 @@ def resolve_browser_profile(watch, datastore) -> BrowserProfile: f"falling back through the chain" ) - return BUILTIN_REQUESTS + return get_default_browser_builtin() diff --git a/changedetectionio/processors/base.py b/changedetectionio/processors/base.py index 19a4d2218..92a0afbc8 100644 --- a/changedetectionio/processors/base.py +++ b/changedetectionio/processors/base.py @@ -177,6 +177,8 @@ class difference_detection_processor(): profile_user_agent=profile.user_agent, ignore_https_errors=profile.ignore_https_errors, locale=profile.locale, + service_workers=profile.service_workers, + extra_delay=profile.extra_delay, ) if self.watch.has_browser_steps: diff --git a/changedetectionio/store/__init__.py b/changedetectionio/store/__init__.py index 0dbd6423a..1f2d0edd5 100644 --- a/changedetectionio/store/__init__.py +++ b/changedetectionio/store/__init__.py @@ -218,6 +218,8 @@ class ChangeDetectionStore(DatastoreUpdatesMixin, FileSavingDataStore): changedetection_json = os.path.join(self.datastore_path, "changedetection.json") changedetection_json_old_schema = os.path.join(self.datastore_path, "url-watches.json") + self.preconfigure_browsers_based_on_env() + if os.path.exists(changedetection_json): # Run schema updates if needed # Pass current schema version from loaded datastore (defaults to 0 if not set) @@ -334,6 +336,38 @@ class ChangeDetectionStore(DatastoreUpdatesMixin, FileSavingDataStore): entity = watch_class(datastore_path=self.datastore_path, __datastore=self, default=entity) return entity + def preconfigure_browsers_based_on_env(self): + """Set browser_connection_url on built-in profiles from environment variables. + + Called once at datastore init (before _load_state). Mutates the module-level + _BUILTINS singletons so the URLs are visible to every profile lookup for the + lifetime of the process without needing to touch persisted settings. + """ + from changedetectionio.model import browser_profile as bp + from changedetectionio.strtobool import strtobool + + playwright_url = os.getenv('PLAYWRIGHT_DRIVER_URL') + if playwright_url: + playwright_url = playwright_url.strip('"') + if strtobool(os.getenv('FAST_PUPPETEER_CHROME_FETCHER', 'False')): + bp.BUILTIN_PUPPETEER.browser_connection_url = playwright_url + else: + bp.BUILTIN_PLAYWRIGHT.browser_connection_url = playwright_url + + webdriver_url = os.getenv('WEBDRIVER_URL') + if webdriver_url: + bp.BUILTIN_SELENIUM.browser_connection_url = webdriver_url.strip('"') + + service_workers = os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow') + bp.BUILTIN_PLAYWRIGHT.service_workers = service_workers + bp.BUILTIN_PUPPETEER.service_workers = service_workers + + extra_delay = int(os.getenv('WEBDRIVER_DELAY_BEFORE_CONTENT_READY', 0)) + bp.BUILTIN_PLAYWRIGHT.extra_delay = extra_delay + bp.BUILTIN_PUPPETEER.extra_delay = extra_delay + bp.BUILTIN_SELENIUM.extra_delay = extra_delay + + # ============================================================================ # FileSavingDataStore Abstract Method Implementations # ============================================================================ diff --git a/changedetectionio/tests/proxy_list/test_multiple_proxy.py b/changedetectionio/tests/proxy_list/test_multiple_proxy.py index dfcef5295..dcca34ba1 100644 --- a/changedetectionio/tests/proxy_list/test_multiple_proxy.py +++ b/changedetectionio/tests/proxy_list/test_multiple_proxy.py @@ -22,7 +22,7 @@ def test_preferred_proxy(client, live_server, measure_memory_usage, datastore_pa url_for("ui.ui_edit.edit_page", uuid="first", unpause_on_save=1), data={ "include_filters": "", - "browser_profile": 'browser_chromeplaywright' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'direct_http_requests', + "browser_profile": "system", "headers": "", "proxy": "proxy-two", "tags": "", diff --git a/changedetectionio/tests/proxy_list/test_proxy_noconnect.py b/changedetectionio/tests/proxy_list/test_proxy_noconnect.py index 111289edc..9ea2ef228 100644 --- a/changedetectionio/tests/proxy_list/test_proxy_noconnect.py +++ b/changedetectionio/tests/proxy_list/test_proxy_noconnect.py @@ -41,7 +41,7 @@ def test_proxy_noconnect_custom(client, live_server, measure_memory_usage, datas options = { "url": test_url, - "browser_profile": "browser_chromeplaywright" if os.getenv('PLAYWRIGHT_DRIVER_URL') or os.getenv("WEBDRIVER_URL") else "direct_http_requests", + "browser_profile": "system", "proxy": "ui-0custom-test-proxy", "time_between_check_use_default": "y", } diff --git a/changedetectionio/tests/proxy_socks5/test_socks5_proxy.py b/changedetectionio/tests/proxy_socks5/test_socks5_proxy.py index 8f7ca0945..06a8931a4 100644 --- a/changedetectionio/tests/proxy_socks5/test_socks5_proxy.py +++ b/changedetectionio/tests/proxy_socks5/test_socks5_proxy.py @@ -60,7 +60,7 @@ def test_socks5(client, live_server, measure_memory_usage, datastore_path): url_for("ui.ui_edit.edit_page", uuid="first", unpause_on_save=1), data={ "include_filters": "", - "browser_profile": 'browser_chromeplaywright' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'direct_http_requests', + "browser_profile": "system", "headers": "", "proxy": "ui-0socks5proxy", "tags": "", diff --git a/changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py b/changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py index 7da056f60..8ef62ee9c 100644 --- a/changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py +++ b/changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py @@ -48,7 +48,7 @@ def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage, url_for("ui.ui_edit.edit_page", uuid="first", unpause_on_save=1), data={ "include_filters": "", - "browser_profile": 'browser_chromeplaywright' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'direct_http_requests', + "browser_profile": "system", "headers": "", "proxy": "socks5proxy", "tags": "", diff --git a/changedetectionio/tests/test_request.py b/changedetectionio/tests/test_request.py index 802ae366c..00b3bdd6c 100644 --- a/changedetectionio/tests/test_request.py +++ b/changedetectionio/tests/test_request.py @@ -35,7 +35,7 @@ def test_headers_in_request(client, live_server, measure_memory_usage, datastore data={ "url": test_url, "tags": "", - "browser_profile": 'browser_chromeplaywright' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'direct_http_requests', + "browser_profile": "system", "headers": "jinja2:{{ 1+1 }}\nxxx:ooo\ncool:yeah\r\ncookie:"+cookie_header, "time_between_check_use_default": "y"}, follow_redirects=True @@ -345,7 +345,7 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage, data={ "url": test_url, "tags": "testtag", - "browser_profile": 'browser_chromeplaywright' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'direct_http_requests', + "browser_profile": "system", "headers": "xxx:ooo\ncool:yeah\r\n", "time_between_check_use_default": "y"}, follow_redirects=True