Merge branch 'master' into add-watch-ui-default-browser

changedetectionio/browser_steps/browser_steps.py

@@ -5,7 +5,7 @@ from random import randint
 from loguru import logger
 
 from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT
-from changedetectionio.content_fetchers.base import manage_user_agent
+from changedetectionio.content_fetchers.base import get_playwright_bypass_csp, manage_user_agent
 from changedetectionio.jinja2_custom import render as jinja_render
 from changedetectionio.validate_url import validate_fetch_url_async
 
@@ -367,7 +367,7 @@ class browsersteps_live_ui(steppable_browser_interface):
         # @todo handle multiple contexts, bind a unique id from the browser on each req?
         self.context = await self.playwright_browser.new_context(
             accept_downloads=False,  # Should never be needed
-            bypass_csp=True,  # This is needed to enable JavaScript execution on GitHub and others
+            bypass_csp=get_playwright_bypass_csp(),
             extra_http_headers=self.headers,
             ignore_https_errors=True,
             proxy=proxy,
@@ -514,4 +514,3 @@ class browsersteps_live_ui(steppable_browser_interface):
             pass
             
         return (screenshot, xpath_data)
-

changedetectionio/content_fetchers/base.py

@@ -4,6 +4,7 @@ from loguru import logger
 from pydantic import BaseModel
 
 from changedetectionio.content_fetchers import BrowserStepsStepException
+from changedetectionio.strtobool import strtobool
 
 
 class FetcherCapabilities(BaseModel):
@@ -27,6 +28,16 @@ class FetcherCapabilities(BaseModel):
         })
 
 
+def get_playwright_bypass_csp():
+    """Return whether Playwright-compatible browser contexts should bypass CSP.
+
+    Bypassing CSP remains enabled by default for backward compatibility. Some
+    remote CDP implementations do not support ``Page.setBypassCSP``; operators
+    can disable the option by setting ``PLAYWRIGHT_BYPASS_CSP=false``.
+    """
+    return strtobool(os.getenv('PLAYWRIGHT_BYPASS_CSP', 'true'))
+
+
 def manage_user_agent(headers, current_ua=''):
     """
     Basic setting of user-agent

changedetectionio/content_fetchers/playwright.py

@@ -8,7 +8,7 @@ from loguru import logger
 
 from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
     SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_MAX_TOTAL_HEIGHT, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, FAVICON_FETCHER_JS
-from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
+from changedetectionio.content_fetchers.base import Fetcher, get_playwright_bypass_csp, manage_user_agent
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable, \
     BrowserStepsStepException
 
@@ -284,7 +284,9 @@ class fetcher(Fetcher):
             # Use the default one configured in the App.py model that's passed from fetch_site_status.py
             context = await browser.new_context(
                 accept_downloads=False,  # Should never be needed
-                bypass_csp=True,  # This is needed to enable JavaScript execution on GitHub and others
+                # Enabled by default because sites such as GitHub need it for injected JavaScript.
+                # Some CDP implementations do not support Page.setBypassCSP, so allow operators to disable it.
+                bypass_csp=get_playwright_bypass_csp(),
                 extra_http_headers=request_headers,
                 ignore_https_errors=True,
                 proxy=self.proxy,
@@ -471,4 +473,3 @@ class PlaywrightFetcherPlugin:
 playwright_plugin = PlaywrightFetcherPlugin()
 
 
-

changedetectionio/content_fetchers/puppeteer.py

@@ -10,11 +10,17 @@ from loguru import logger
 from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
     SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_DEFAULT_QUALITY, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, \
     SCREENSHOT_MAX_TOTAL_HEIGHT, FAVICON_FETCHER_JS
-from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
+from changedetectionio.content_fetchers.base import Fetcher, get_playwright_bypass_csp, manage_user_agent
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, \
     BrowserConnectError
 
 
+async def _configure_puppeteer_csp(page):
+    """Enable CSP bypass without requiring unsupported CDP methods when disabled."""
+    if get_playwright_bypass_csp():
+        await page.setBypassCSP(True)
+
+
 # Bug 3 in Playwright screenshot handling
 # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
 
@@ -347,7 +353,7 @@ class fetcher(Fetcher):
             # Attempt to strip 'HeadlessChrome' etc
             await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent')))
 
-        await self.page.setBypassCSP(True)
+        await _configure_puppeteer_csp(self.page)
         if request_headers:
             await self.page.setExtraHTTPHeaders(request_headers)
 

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -108,7 +108,14 @@ async () => {
             '品切れ',
             '已售',
             '已售完',
-            '품절'
+            '품절',
+            "товар закінчився",
+            "немає в наявності",
+            "нема в наявності",
+            "закінчився",
+            "знято з виробництва",
+            "недоступний",
+            "немає на складі"
         ];
 
 

changedetectionio/processors/restock_diff/processor.py

@@ -364,12 +364,8 @@ def get_itemprop_availability(html_content) -> Restock:
         if availability_result:
             value['availability'] = availability_result[0].value
 
-        if value.get('availability'):
-            value['availability'] = re.sub(r'(?i)^(https|http)://schema.org/', '',
-                                           value.get('availability').strip(' "\'').lower()) if value.get('availability') else None
-
         # Second, go dig OpenGraph which is something that jsonpath_ng cant do because of the tuples and double-dots (:)
-        if not value.get('price') or value.get('availability'):
+        if not value.get('price') or not value.get('availability') or not value.get('currency'):
             logger.debug("Alternatively digging through OpenGraph properties for restock/price info..")
             jsonpath_expr = parse('$..properties')
 
@@ -380,6 +376,14 @@ def get_itemprop_availability(html_content) -> Restock:
                     value['availability'] = _search_prop_by_value([match.value], "product:availability")
                 if not value.get('currency'):
                     value['currency'] = _search_prop_by_value([match.value], "price:currency")
+
+        # Normalise after both sources have been tried, otherwise an availability that came from
+        # OpenGraph stays raw. The OpenGraph vocabulary spells it "in stock" while the in-stock
+        # matcher looks for "instock", so the spaces have to go too.
+        if value.get('availability'):
+            value['availability'] = re.sub(r'(?i)^(https|http)://schema.org/', '',
+                                           value.get('availability').strip(' "\'').lower())
+            value['availability'] = re.sub(r'\s+', '', value['availability'])
     logger.trace(f"Processed with Extruct in {time.time()-now:.3f}s")
 
     return value

changedetectionio/tests/unit/test_playwright_bypass_csp.py

@@ -0,0 +1,154 @@
+import asyncio
+import sys
+from types import ModuleType, SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+from changedetectionio.browser_steps.browser_steps import browsersteps_live_ui
+from changedetectionio.content_fetchers.base import get_playwright_bypass_csp
+
+
+def test_playwright_bypass_csp_defaults_to_enabled(monkeypatch):
+    monkeypatch.delenv('PLAYWRIGHT_BYPASS_CSP', raising=False)
+
+    assert get_playwright_bypass_csp() is True
+
+
+@pytest.mark.parametrize(
+    ('configured_value', 'expected'),
+    [
+        ('true', True),
+        ('1', True),
+        ('yes', True),
+        ('false', False),
+        ('0', False),
+        ('no', False),
+    ],
+)
+def test_playwright_bypass_csp_parses_boolean_environment_values(
+    monkeypatch, configured_value, expected
+):
+    monkeypatch.setenv('PLAYWRIGHT_BYPASS_CSP', configured_value)
+
+    assert get_playwright_bypass_csp() is expected
+
+
+@pytest.mark.parametrize('configured_value', ('true', 'false'))
+def test_playwright_fetcher_passes_bypass_csp_to_browser_context(monkeypatch, configured_value):
+    monkeypatch.setenv('PLAYWRIGHT_BYPASS_CSP', configured_value)
+
+    class ContextCreationStopped(Exception):
+        pass
+
+    browser = SimpleNamespace(
+        new_context=AsyncMock(side_effect=ContextCreationStopped),
+    )
+    browser_type = SimpleNamespace(
+        connect_over_cdp=AsyncMock(return_value=browser),
+    )
+
+    class AsyncPlaywrightContextManager:
+        async def __aenter__(self):
+            return SimpleNamespace(chromium=browser_type)
+
+        async def __aexit__(self, exc_type, exc_value, traceback):
+            return False
+
+    async_api_module = ModuleType('playwright.async_api')
+    async_api_module.async_playwright = AsyncPlaywrightContextManager
+
+    errors_module = ModuleType('playwright._impl._errors')
+    errors_module.TimeoutError = TimeoutError
+    impl_module = ModuleType('playwright._impl')
+    impl_module._errors = errors_module
+    playwright_module = ModuleType('playwright')
+    playwright_module._impl = impl_module
+
+    monkeypatch.setitem(sys.modules, 'playwright', playwright_module)
+    monkeypatch.setitem(sys.modules, 'playwright.async_api', async_api_module)
+    monkeypatch.setitem(sys.modules, 'playwright._impl', impl_module)
+    monkeypatch.setitem(sys.modules, 'playwright._impl._errors', errors_module)
+
+    from changedetectionio.content_fetchers.playwright import fetcher
+
+    with pytest.raises(ContextCreationStopped):
+        asyncio.run(fetcher().run(request_headers={}, url='https://example.com'))
+
+    assert browser.new_context.await_args.kwargs['bypass_csp'] is (configured_value == 'true')
+
+
+@pytest.mark.parametrize('configured_value', ('true', 'false'))
+def test_browser_steps_passes_bypass_csp_to_browser_context(monkeypatch, configured_value):
+    monkeypatch.setenv('PLAYWRIGHT_BYPASS_CSP', configured_value)
+
+    page = Mock()
+    page.wait_for_timeout = AsyncMock()
+    context = SimpleNamespace(new_page=AsyncMock(return_value=page))
+    browser = SimpleNamespace(new_context=AsyncMock(return_value=context))
+    browser_steps = browsersteps_live_ui(
+        playwright_browser=browser, start_url='https://example.com'
+    )
+
+    asyncio.run(browser_steps.connect())
+
+    assert browser.new_context.await_args.kwargs['bypass_csp'] is (configured_value == 'true')
+
+
+@pytest.mark.parametrize('configured_value', ('true', 'false'))
+def test_puppeteer_only_sends_set_bypass_csp_when_enabled(monkeypatch, configured_value):
+    monkeypatch.setenv('PLAYWRIGHT_BYPASS_CSP', configured_value)
+
+    from changedetectionio.content_fetchers.puppeteer import _configure_puppeteer_csp
+
+    page = SimpleNamespace(setBypassCSP=AsyncMock())
+    asyncio.run(_configure_puppeteer_csp(page))
+
+    if configured_value == 'true':
+        page.setBypassCSP.assert_awaited_once_with(True)
+    else:
+        page.setBypassCSP.assert_not_awaited()
+
+
+def test_puppeteer_fetcher_configures_csp_on_created_page(monkeypatch):
+    from changedetectionio.content_fetchers import puppeteer
+
+    class CSPConfigurationReached(Exception):
+        pass
+
+    page = SimpleNamespace(
+        evaluate=AsyncMock(return_value='Mozilla/5.0'),
+        setUserAgent=AsyncMock(),
+    )
+    browser = SimpleNamespace(newPage=AsyncMock(return_value=page))
+    pyppeteer_instance = SimpleNamespace(connect=AsyncMock(return_value=browser))
+
+    pyppeteer_module = ModuleType('pyppeteer')
+    pyppeteer_module.Pyppeteer = Mock(return_value=pyppeteer_instance)
+    stealth_module = ModuleType('pyppeteerstealth')
+    stealth_module.inject_evasions_into_page = AsyncMock()
+    monkeypatch.setitem(sys.modules, 'pyppeteer', pyppeteer_module)
+    monkeypatch.setitem(sys.modules, 'pyppeteerstealth', stealth_module)
+
+    configure_csp = AsyncMock(side_effect=CSPConfigurationReached)
+    monkeypatch.setattr(puppeteer, '_configure_puppeteer_csp', configure_csp)
+
+    with pytest.raises(CSPConfigurationReached):
+        asyncio.run(
+            puppeteer.fetcher().fetch_page(
+                current_include_filters=None,
+                empty_pages_are_a_change=False,
+                fetch_favicon=False,
+                ignore_status_codes=False,
+                is_binary=False,
+                request_body=None,
+                request_headers={},
+                request_method='GET',
+                screenshot_format=None,
+                timeout=45,
+                url='https://example.com',
+                watch_uuid='test-watch',
+            )
+        )
+
+    configure_csp.assert_awaited_once_with(page)

changedetectionio/tests/unit/test_restock_logic.py

@@ -17,5 +17,32 @@ class TestDiffBuilder(unittest.TestCase):
         assert restock_diff.is_between(number=10, lower=None, upper=11) == True, "Between None and 11"
         assert not restock_diff.is_between(number=12, lower=None, upper=11) == True, "12 is not between None and 11"
 
+    def test_itemprop_availability_opengraph_fallback(self):
+        """Availability/currency missing from JSON-LD should be filled from OpenGraph
+        (Facebook commerce 'product:availability' / 'product:price:currency' meta tags)."""
+        html_content = """<!DOCTYPE html>
+        <html prefix="og: https://ogp.me/ns# product: https://ogp.me/ns/product#">
+        <head>
+        <meta property="og:type" content="product">
+        <meta property="og:title" content="Some Product">
+        <meta property="product:availability" content="in stock">
+        <meta property="product:price:currency" content="EUR">
+        <script type="application/ld+json">
+        {"@context": "https://schema.org", "@type": "Product", "name": "Some Product",
+         "offers": {"@type": "Offer", "price": "155.55"}}
+        </script>
+        </head>
+        <body><h1>Some Product</h1></body>
+        </html>"""
+
+        value = restock_diff.get_itemprop_availability(html_content)
+        assert value.get('price') == 155.55, "price should be found via JSON-LD"
+        assert value.get('currency') == 'EUR', "currency should be found via OpenGraph fallback"
+        # Normalised the same way a schema.org value is, so that the in-stock matcher
+        # (which looks for 'instock') sees it. OpenGraph spells it 'in stock'.
+        assert value.get('availability') == 'instock', "availability should be found via OpenGraph fallback and normalised"
+        assert any(s in value.get('availability') for s in ['instock', 'instoreonly']), \
+            "a product advertised as available must read as in stock, not out of stock"
+
 if __name__ == '__main__':
     unittest.main()

changedetectionio/translations/messages.pot

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: changedetection.io 0.55.8\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2026-08-25 17:06+0200\n"
+"POT-Creation-Date: 2026-08-26 11:14+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

docker-compose.yml

@@ -26,6 +26,8 @@ services:
   #
   #       Uncomment below and the "sockpuppetbrowser" to use a real Chrome browser (It uses the "playwright" protocol)
   #      - PLAYWRIGHT_DRIVER_URL=ws://browser-sockpuppet-chrome:3000
+  #      # Disable when the remote CDP implementation does not support `Page.setBypassCSP` - like with Obscura browser.
+  #      - PLAYWRIGHT_BYPASS_CSP=false
   #
   #
   #       Alternative WebDriver/selenium URL, do not use "'s or 's! (old, deprecated, does not support screenshots very well, Can't handle custom headers etc)
@@ -158,4 +160,3 @@ services:
 
 volumes:
   changedetection-data:
-