feat: make Playwright CSP bypass configurable (#4298)

changedetectionio/browser_steps/browser_steps.py

@@ -5,7 +5,7 @@ from random import randint
 from loguru import logger
 
 from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT
-from changedetectionio.content_fetchers.base import manage_user_agent
+from changedetectionio.content_fetchers.base import get_playwright_bypass_csp, manage_user_agent
 from changedetectionio.jinja2_custom import render as jinja_render
 from changedetectionio.validate_url import validate_fetch_url_async
 
@@ -367,7 +367,7 @@ class browsersteps_live_ui(steppable_browser_interface):
         # @todo handle multiple contexts, bind a unique id from the browser on each req?
         self.context = await self.playwright_browser.new_context(
             accept_downloads=False,  # Should never be needed
-            bypass_csp=True,  # This is needed to enable JavaScript execution on GitHub and others
+            bypass_csp=get_playwright_bypass_csp(),
             extra_http_headers=self.headers,
             ignore_https_errors=True,
             proxy=proxy,
@@ -514,4 +514,3 @@ class browsersteps_live_ui(steppable_browser_interface):
             pass
             
         return (screenshot, xpath_data)
-

changedetectionio/content_fetchers/base.py

@@ -4,6 +4,7 @@ from loguru import logger
 from pydantic import BaseModel
 
 from changedetectionio.content_fetchers import BrowserStepsStepException
+from changedetectionio.strtobool import strtobool
 
 
 class FetcherCapabilities(BaseModel):
@@ -27,6 +28,16 @@ class FetcherCapabilities(BaseModel):
         })
 
 
+def get_playwright_bypass_csp():
+    """Return whether Playwright-compatible browser contexts should bypass CSP.
+
+    Bypassing CSP remains enabled by default for backward compatibility. Some
+    remote CDP implementations do not support ``Page.setBypassCSP``; operators
+    can disable the option by setting ``PLAYWRIGHT_BYPASS_CSP=false``.
+    """
+    return strtobool(os.getenv('PLAYWRIGHT_BYPASS_CSP', 'true'))
+
+
 def manage_user_agent(headers, current_ua=''):
     """
     Basic setting of user-agent

changedetectionio/content_fetchers/playwright.py

@@ -8,7 +8,7 @@ from loguru import logger
 
 from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
     SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_MAX_TOTAL_HEIGHT, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, FAVICON_FETCHER_JS
-from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
+from changedetectionio.content_fetchers.base import Fetcher, get_playwright_bypass_csp, manage_user_agent
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable, \
     BrowserStepsStepException
 
@@ -284,7 +284,9 @@ class fetcher(Fetcher):
             # Use the default one configured in the App.py model that's passed from fetch_site_status.py
             context = await browser.new_context(
                 accept_downloads=False,  # Should never be needed
-                bypass_csp=True,  # This is needed to enable JavaScript execution on GitHub and others
+                # Enabled by default because sites such as GitHub need it for injected JavaScript.
+                # Some CDP implementations do not support Page.setBypassCSP, so allow operators to disable it.
+                bypass_csp=get_playwright_bypass_csp(),
                 extra_http_headers=request_headers,
                 ignore_https_errors=True,
                 proxy=self.proxy,
@@ -471,4 +473,3 @@ class PlaywrightFetcherPlugin:
 playwright_plugin = PlaywrightFetcherPlugin()
 
 
-

changedetectionio/content_fetchers/puppeteer.py

@@ -10,11 +10,17 @@ from loguru import logger
 from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
     SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_DEFAULT_QUALITY, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, \
     SCREENSHOT_MAX_TOTAL_HEIGHT, FAVICON_FETCHER_JS
-from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
+from changedetectionio.content_fetchers.base import Fetcher, get_playwright_bypass_csp, manage_user_agent
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, \
     BrowserConnectError
 
 
+async def _configure_puppeteer_csp(page):
+    """Enable CSP bypass without requiring unsupported CDP methods when disabled."""
+    if get_playwright_bypass_csp():
+        await page.setBypassCSP(True)
+
+
 # Bug 3 in Playwright screenshot handling
 # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
 
@@ -347,7 +353,7 @@ class fetcher(Fetcher):
             # Attempt to strip 'HeadlessChrome' etc
             await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent')))
 
-        await self.page.setBypassCSP(True)
+        await _configure_puppeteer_csp(self.page)
         if request_headers:
             await self.page.setExtraHTTPHeaders(request_headers)
 

changedetectionio/tests/unit/test_playwright_bypass_csp.py

@@ -0,0 +1,154 @@
+import asyncio
+import sys
+from types import ModuleType, SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+from changedetectionio.browser_steps.browser_steps import browsersteps_live_ui
+from changedetectionio.content_fetchers.base import get_playwright_bypass_csp
+
+
+def test_playwright_bypass_csp_defaults_to_enabled(monkeypatch):
+    monkeypatch.delenv('PLAYWRIGHT_BYPASS_CSP', raising=False)
+
+    assert get_playwright_bypass_csp() is True
+
+
+@pytest.mark.parametrize(
+    ('configured_value', 'expected'),
+    [
+        ('true', True),
+        ('1', True),
+        ('yes', True),
+        ('false', False),
+        ('0', False),
+        ('no', False),
+    ],
+)
+def test_playwright_bypass_csp_parses_boolean_environment_values(
+    monkeypatch, configured_value, expected
+):
+    monkeypatch.setenv('PLAYWRIGHT_BYPASS_CSP', configured_value)
+
+    assert get_playwright_bypass_csp() is expected
+
+
+@pytest.mark.parametrize('configured_value', ('true', 'false'))
+def test_playwright_fetcher_passes_bypass_csp_to_browser_context(monkeypatch, configured_value):
+    monkeypatch.setenv('PLAYWRIGHT_BYPASS_CSP', configured_value)
+
+    class ContextCreationStopped(Exception):
+        pass
+
+    browser = SimpleNamespace(
+        new_context=AsyncMock(side_effect=ContextCreationStopped),
+    )
+    browser_type = SimpleNamespace(
+        connect_over_cdp=AsyncMock(return_value=browser),
+    )
+
+    class AsyncPlaywrightContextManager:
+        async def __aenter__(self):
+            return SimpleNamespace(chromium=browser_type)
+
+        async def __aexit__(self, exc_type, exc_value, traceback):
+            return False
+
+    async_api_module = ModuleType('playwright.async_api')
+    async_api_module.async_playwright = AsyncPlaywrightContextManager
+
+    errors_module = ModuleType('playwright._impl._errors')
+    errors_module.TimeoutError = TimeoutError
+    impl_module = ModuleType('playwright._impl')
+    impl_module._errors = errors_module
+    playwright_module = ModuleType('playwright')
+    playwright_module._impl = impl_module
+
+    monkeypatch.setitem(sys.modules, 'playwright', playwright_module)
+    monkeypatch.setitem(sys.modules, 'playwright.async_api', async_api_module)
+    monkeypatch.setitem(sys.modules, 'playwright._impl', impl_module)
+    monkeypatch.setitem(sys.modules, 'playwright._impl._errors', errors_module)
+
+    from changedetectionio.content_fetchers.playwright import fetcher
+
+    with pytest.raises(ContextCreationStopped):
+        asyncio.run(fetcher().run(request_headers={}, url='https://example.com'))
+
+    assert browser.new_context.await_args.kwargs['bypass_csp'] is (configured_value == 'true')
+
+
+@pytest.mark.parametrize('configured_value', ('true', 'false'))
+def test_browser_steps_passes_bypass_csp_to_browser_context(monkeypatch, configured_value):
+    monkeypatch.setenv('PLAYWRIGHT_BYPASS_CSP', configured_value)
+
+    page = Mock()
+    page.wait_for_timeout = AsyncMock()
+    context = SimpleNamespace(new_page=AsyncMock(return_value=page))
+    browser = SimpleNamespace(new_context=AsyncMock(return_value=context))
+    browser_steps = browsersteps_live_ui(
+        playwright_browser=browser, start_url='https://example.com'
+    )
+
+    asyncio.run(browser_steps.connect())
+
+    assert browser.new_context.await_args.kwargs['bypass_csp'] is (configured_value == 'true')
+
+
+@pytest.mark.parametrize('configured_value', ('true', 'false'))
+def test_puppeteer_only_sends_set_bypass_csp_when_enabled(monkeypatch, configured_value):
+    monkeypatch.setenv('PLAYWRIGHT_BYPASS_CSP', configured_value)
+
+    from changedetectionio.content_fetchers.puppeteer import _configure_puppeteer_csp
+
+    page = SimpleNamespace(setBypassCSP=AsyncMock())
+    asyncio.run(_configure_puppeteer_csp(page))
+
+    if configured_value == 'true':
+        page.setBypassCSP.assert_awaited_once_with(True)
+    else:
+        page.setBypassCSP.assert_not_awaited()
+
+
+def test_puppeteer_fetcher_configures_csp_on_created_page(monkeypatch):
+    from changedetectionio.content_fetchers import puppeteer
+
+    class CSPConfigurationReached(Exception):
+        pass
+
+    page = SimpleNamespace(
+        evaluate=AsyncMock(return_value='Mozilla/5.0'),
+        setUserAgent=AsyncMock(),
+    )
+    browser = SimpleNamespace(newPage=AsyncMock(return_value=page))
+    pyppeteer_instance = SimpleNamespace(connect=AsyncMock(return_value=browser))
+
+    pyppeteer_module = ModuleType('pyppeteer')
+    pyppeteer_module.Pyppeteer = Mock(return_value=pyppeteer_instance)
+    stealth_module = ModuleType('pyppeteerstealth')
+    stealth_module.inject_evasions_into_page = AsyncMock()
+    monkeypatch.setitem(sys.modules, 'pyppeteer', pyppeteer_module)
+    monkeypatch.setitem(sys.modules, 'pyppeteerstealth', stealth_module)
+
+    configure_csp = AsyncMock(side_effect=CSPConfigurationReached)
+    monkeypatch.setattr(puppeteer, '_configure_puppeteer_csp', configure_csp)
+
+    with pytest.raises(CSPConfigurationReached):
+        asyncio.run(
+            puppeteer.fetcher().fetch_page(
+                current_include_filters=None,
+                empty_pages_are_a_change=False,
+                fetch_favicon=False,
+                ignore_status_codes=False,
+                is_binary=False,
+                request_body=None,
+                request_headers={},
+                request_method='GET',
+                screenshot_format=None,
+                timeout=45,
+                url='https://example.com',
+                watch_uuid='test-watch',
+            )
+        )
+
+    configure_csp.assert_awaited_once_with(page)

docker-compose.yml

@@ -26,6 +26,8 @@ services:
   #
   #       Uncomment below and the "sockpuppetbrowser" to use a real Chrome browser (It uses the "playwright" protocol)
   #      - PLAYWRIGHT_DRIVER_URL=ws://browser-sockpuppet-chrome:3000
+  #      # Disable when the remote CDP implementation does not support `Page.setBypassCSP` - like with Obscura browser.
+  #      - PLAYWRIGHT_BYPASS_CSP=false
   #
   #
   #       Alternative WebDriver/selenium URL, do not use "'s or 's! (old, deprecated, does not support screenshots very well, Can't handle custom headers etc)
@@ -158,4 +160,3 @@ services:
 
 volumes:
   changedetection-data:
-