From 465ff40bdeaba3ce5e4d7f14290b0496a97178f5 Mon Sep 17 00:00:00 2001 From: "Karl Q." Date: Tue, 25 Aug 2026 13:24:10 -0700 Subject: [PATCH] feat: make Playwright CSP bypass configurable (#4298) --- .../browser_steps/browser_steps.py | 5 +- changedetectionio/content_fetchers/base.py | 11 ++ .../content_fetchers/playwright.py | 7 +- .../content_fetchers/puppeteer.py | 10 +- .../tests/unit/test_playwright_bypass_csp.py | 154 ++++++++++++++++++ docker-compose.yml | 3 +- 6 files changed, 181 insertions(+), 9 deletions(-) create mode 100644 changedetectionio/tests/unit/test_playwright_bypass_csp.py diff --git a/changedetectionio/browser_steps/browser_steps.py b/changedetectionio/browser_steps/browser_steps.py index 062a7b88..3aace66d 100644 --- a/changedetectionio/browser_steps/browser_steps.py +++ b/changedetectionio/browser_steps/browser_steps.py @@ -5,7 +5,7 @@ from random import randint from loguru import logger from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT -from changedetectionio.content_fetchers.base import manage_user_agent +from changedetectionio.content_fetchers.base import get_playwright_bypass_csp, manage_user_agent from changedetectionio.jinja2_custom import render as jinja_render from changedetectionio.validate_url import validate_fetch_url_async @@ -367,7 +367,7 @@ class browsersteps_live_ui(steppable_browser_interface): # @todo handle multiple contexts, bind a unique id from the browser on each req? self.context = await self.playwright_browser.new_context( accept_downloads=False, # Should never be needed - bypass_csp=True, # This is needed to enable JavaScript execution on GitHub and others + bypass_csp=get_playwright_bypass_csp(), extra_http_headers=self.headers, ignore_https_errors=True, proxy=proxy, @@ -514,4 +514,3 @@ class browsersteps_live_ui(steppable_browser_interface): pass return (screenshot, xpath_data) - diff --git a/changedetectionio/content_fetchers/base.py b/changedetectionio/content_fetchers/base.py index f4cd459f..079bed19 100644 --- a/changedetectionio/content_fetchers/base.py +++ b/changedetectionio/content_fetchers/base.py @@ -4,6 +4,7 @@ from loguru import logger from pydantic import BaseModel from changedetectionio.content_fetchers import BrowserStepsStepException +from changedetectionio.strtobool import strtobool class FetcherCapabilities(BaseModel): @@ -27,6 +28,16 @@ class FetcherCapabilities(BaseModel): }) +def get_playwright_bypass_csp(): + """Return whether Playwright-compatible browser contexts should bypass CSP. + + Bypassing CSP remains enabled by default for backward compatibility. Some + remote CDP implementations do not support ``Page.setBypassCSP``; operators + can disable the option by setting ``PLAYWRIGHT_BYPASS_CSP=false``. + """ + return strtobool(os.getenv('PLAYWRIGHT_BYPASS_CSP', 'true')) + + def manage_user_agent(headers, current_ua=''): """ Basic setting of user-agent diff --git a/changedetectionio/content_fetchers/playwright.py b/changedetectionio/content_fetchers/playwright.py index 6c5256b4..f20981df 100644 --- a/changedetectionio/content_fetchers/playwright.py +++ b/changedetectionio/content_fetchers/playwright.py @@ -8,7 +8,7 @@ from loguru import logger from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \ SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_MAX_TOTAL_HEIGHT, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, FAVICON_FETCHER_JS -from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent +from changedetectionio.content_fetchers.base import Fetcher, get_playwright_bypass_csp, manage_user_agent from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable, \ BrowserStepsStepException @@ -284,7 +284,9 @@ class fetcher(Fetcher): # Use the default one configured in the App.py model that's passed from fetch_site_status.py context = await browser.new_context( accept_downloads=False, # Should never be needed - bypass_csp=True, # This is needed to enable JavaScript execution on GitHub and others + # Enabled by default because sites such as GitHub need it for injected JavaScript. + # Some CDP implementations do not support Page.setBypassCSP, so allow operators to disable it. + bypass_csp=get_playwright_bypass_csp(), extra_http_headers=request_headers, ignore_https_errors=True, proxy=self.proxy, @@ -471,4 +473,3 @@ class PlaywrightFetcherPlugin: playwright_plugin = PlaywrightFetcherPlugin() - diff --git a/changedetectionio/content_fetchers/puppeteer.py b/changedetectionio/content_fetchers/puppeteer.py index 39699645..849c21ed 100644 --- a/changedetectionio/content_fetchers/puppeteer.py +++ b/changedetectionio/content_fetchers/puppeteer.py @@ -10,11 +10,17 @@ from loguru import logger from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \ SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_DEFAULT_QUALITY, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, \ SCREENSHOT_MAX_TOTAL_HEIGHT, FAVICON_FETCHER_JS -from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent +from changedetectionio.content_fetchers.base import Fetcher, get_playwright_bypass_csp, manage_user_agent from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, \ BrowserConnectError +async def _configure_puppeteer_csp(page): + """Enable CSP bypass without requiring unsupported CDP methods when disabled.""" + if get_playwright_bypass_csp(): + await page.setBypassCSP(True) + + # Bug 3 in Playwright screenshot handling # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it @@ -347,7 +353,7 @@ class fetcher(Fetcher): # Attempt to strip 'HeadlessChrome' etc await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent'))) - await self.page.setBypassCSP(True) + await _configure_puppeteer_csp(self.page) if request_headers: await self.page.setExtraHTTPHeaders(request_headers) diff --git a/changedetectionio/tests/unit/test_playwright_bypass_csp.py b/changedetectionio/tests/unit/test_playwright_bypass_csp.py new file mode 100644 index 00000000..4fe7893d --- /dev/null +++ b/changedetectionio/tests/unit/test_playwright_bypass_csp.py @@ -0,0 +1,154 @@ +import asyncio +import sys +from types import ModuleType, SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest + +from changedetectionio.browser_steps.browser_steps import browsersteps_live_ui +from changedetectionio.content_fetchers.base import get_playwright_bypass_csp + + +def test_playwright_bypass_csp_defaults_to_enabled(monkeypatch): + monkeypatch.delenv('PLAYWRIGHT_BYPASS_CSP', raising=False) + + assert get_playwright_bypass_csp() is True + + +@pytest.mark.parametrize( + ('configured_value', 'expected'), + [ + ('true', True), + ('1', True), + ('yes', True), + ('false', False), + ('0', False), + ('no', False), + ], +) +def test_playwright_bypass_csp_parses_boolean_environment_values( + monkeypatch, configured_value, expected +): + monkeypatch.setenv('PLAYWRIGHT_BYPASS_CSP', configured_value) + + assert get_playwright_bypass_csp() is expected + + +@pytest.mark.parametrize('configured_value', ('true', 'false')) +def test_playwright_fetcher_passes_bypass_csp_to_browser_context(monkeypatch, configured_value): + monkeypatch.setenv('PLAYWRIGHT_BYPASS_CSP', configured_value) + + class ContextCreationStopped(Exception): + pass + + browser = SimpleNamespace( + new_context=AsyncMock(side_effect=ContextCreationStopped), + ) + browser_type = SimpleNamespace( + connect_over_cdp=AsyncMock(return_value=browser), + ) + + class AsyncPlaywrightContextManager: + async def __aenter__(self): + return SimpleNamespace(chromium=browser_type) + + async def __aexit__(self, exc_type, exc_value, traceback): + return False + + async_api_module = ModuleType('playwright.async_api') + async_api_module.async_playwright = AsyncPlaywrightContextManager + + errors_module = ModuleType('playwright._impl._errors') + errors_module.TimeoutError = TimeoutError + impl_module = ModuleType('playwright._impl') + impl_module._errors = errors_module + playwright_module = ModuleType('playwright') + playwright_module._impl = impl_module + + monkeypatch.setitem(sys.modules, 'playwright', playwright_module) + monkeypatch.setitem(sys.modules, 'playwright.async_api', async_api_module) + monkeypatch.setitem(sys.modules, 'playwright._impl', impl_module) + monkeypatch.setitem(sys.modules, 'playwright._impl._errors', errors_module) + + from changedetectionio.content_fetchers.playwright import fetcher + + with pytest.raises(ContextCreationStopped): + asyncio.run(fetcher().run(request_headers={}, url='https://example.com')) + + assert browser.new_context.await_args.kwargs['bypass_csp'] is (configured_value == 'true') + + +@pytest.mark.parametrize('configured_value', ('true', 'false')) +def test_browser_steps_passes_bypass_csp_to_browser_context(monkeypatch, configured_value): + monkeypatch.setenv('PLAYWRIGHT_BYPASS_CSP', configured_value) + + page = Mock() + page.wait_for_timeout = AsyncMock() + context = SimpleNamespace(new_page=AsyncMock(return_value=page)) + browser = SimpleNamespace(new_context=AsyncMock(return_value=context)) + browser_steps = browsersteps_live_ui( + playwright_browser=browser, start_url='https://example.com' + ) + + asyncio.run(browser_steps.connect()) + + assert browser.new_context.await_args.kwargs['bypass_csp'] is (configured_value == 'true') + + +@pytest.mark.parametrize('configured_value', ('true', 'false')) +def test_puppeteer_only_sends_set_bypass_csp_when_enabled(monkeypatch, configured_value): + monkeypatch.setenv('PLAYWRIGHT_BYPASS_CSP', configured_value) + + from changedetectionio.content_fetchers.puppeteer import _configure_puppeteer_csp + + page = SimpleNamespace(setBypassCSP=AsyncMock()) + asyncio.run(_configure_puppeteer_csp(page)) + + if configured_value == 'true': + page.setBypassCSP.assert_awaited_once_with(True) + else: + page.setBypassCSP.assert_not_awaited() + + +def test_puppeteer_fetcher_configures_csp_on_created_page(monkeypatch): + from changedetectionio.content_fetchers import puppeteer + + class CSPConfigurationReached(Exception): + pass + + page = SimpleNamespace( + evaluate=AsyncMock(return_value='Mozilla/5.0'), + setUserAgent=AsyncMock(), + ) + browser = SimpleNamespace(newPage=AsyncMock(return_value=page)) + pyppeteer_instance = SimpleNamespace(connect=AsyncMock(return_value=browser)) + + pyppeteer_module = ModuleType('pyppeteer') + pyppeteer_module.Pyppeteer = Mock(return_value=pyppeteer_instance) + stealth_module = ModuleType('pyppeteerstealth') + stealth_module.inject_evasions_into_page = AsyncMock() + monkeypatch.setitem(sys.modules, 'pyppeteer', pyppeteer_module) + monkeypatch.setitem(sys.modules, 'pyppeteerstealth', stealth_module) + + configure_csp = AsyncMock(side_effect=CSPConfigurationReached) + monkeypatch.setattr(puppeteer, '_configure_puppeteer_csp', configure_csp) + + with pytest.raises(CSPConfigurationReached): + asyncio.run( + puppeteer.fetcher().fetch_page( + current_include_filters=None, + empty_pages_are_a_change=False, + fetch_favicon=False, + ignore_status_codes=False, + is_binary=False, + request_body=None, + request_headers={}, + request_method='GET', + screenshot_format=None, + timeout=45, + url='https://example.com', + watch_uuid='test-watch', + ) + ) + + configure_csp.assert_awaited_once_with(page) diff --git a/docker-compose.yml b/docker-compose.yml index d6defee4..00ec49f8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,8 @@ services: # # Uncomment below and the "sockpuppetbrowser" to use a real Chrome browser (It uses the "playwright" protocol) # - PLAYWRIGHT_DRIVER_URL=ws://browser-sockpuppet-chrome:3000 + # # Disable when the remote CDP implementation does not support `Page.setBypassCSP` - like with Obscura browser. + # - PLAYWRIGHT_BYPASS_CSP=false # # # Alternative WebDriver/selenium URL, do not use "'s or 's! (old, deprecated, does not support screenshots very well, Can't handle custom headers etc) @@ -158,4 +160,3 @@ services: volumes: changedetection-data: -