changedetection.io/changedetectionio/content_fetchers/playwright.py

import asyncio
import gc
import json
import os
from urllib.parse import urlparse

from loguru import logger

from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
    SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_MAX_TOTAL_HEIGHT, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, FAVICON_FETCHER_JS
from changedetectionio.content_fetchers.base import Fetcher, get_playwright_bypass_csp, manage_user_agent
from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable, \
    BrowserStepsStepException


async def capture_full_page_async(page, screenshot_format='JPEG', watch_uuid=None, lock_viewport_elements=False):
    import os
    import time

    start = time.time()
    watch_info = f"[{watch_uuid}] " if watch_uuid else ""

    setup_start = time.time()
    page_height = await page.evaluate("document.documentElement.scrollHeight")
    page_width = await page.evaluate("document.documentElement.scrollWidth")
    original_viewport = page.viewport_size
    dimensions_time = time.time() - setup_start

    logger.debug(f"{watch_info}Playwright viewport size {page.viewport_size} page height {page_height} page width {page_width} (got dimensions in {dimensions_time:.2f}s)")

    # Use an approach similar to puppeteer: set a larger viewport and take screenshots in chunks
    step_size = SCREENSHOT_SIZE_STITCH_THRESHOLD # Size that won't cause GPU to overflow
    screenshot_chunks = []
    y = 0
    elements_locked = False

    # Only lock viewport elements if explicitly enabled (for image_ssim_diff processor)
    # This prevents headers/ads from resizing when viewport changes
    if lock_viewport_elements and page_height > page.viewport_size['height']:
        lock_start = time.time()
        lock_elements_js_path = os.path.join(os.path.dirname(__file__), 'res', 'lock-elements-sizing.js')
        with open(lock_elements_js_path, 'r') as f:
            lock_elements_js = f.read()
        await page.evaluate(lock_elements_js)
        elements_locked = True
        lock_time = time.time() - lock_start
        logger.debug(f"{watch_info}Viewport element locking enabled (took {lock_time:.2f}s)")

    if page_height > page.viewport_size['height']:
        if page_height < step_size:
            step_size = page_height # Incase page is bigger than default viewport but smaller than proposed step size
        # Never set viewport taller than our max capture height - otherwise one screenshot chunk
        # captures the whole (e.g. 8098px) page even when SCREENSHOT_MAX_HEIGHT=1000
        step_size = min(step_size, SCREENSHOT_MAX_TOTAL_HEIGHT)
        viewport_start = time.time()
        logger.debug(f"{watch_info}Setting bigger viewport to step through large page width W{page.viewport_size['width']}xH{step_size} because page_height > viewport_size")
        # Set viewport to a larger size to capture more content at once
        await page.set_viewport_size({'width': page.viewport_size['width'], 'height': step_size})
        viewport_time = time.time() - viewport_start
        logger.debug(f"{watch_info}Viewport changed to {page.viewport_size['width']}x{step_size} (took {viewport_time:.2f}s)")

    # Capture screenshots in chunks up to the max total height
    capture_start = time.time()
    chunk_times = []
    # Use PNG for better quality (no compression artifacts), JPEG for smaller size
    screenshot_type = screenshot_format.lower() if screenshot_format else 'jpeg'
    # PNG should use quality 100, JPEG uses configurable quality
    screenshot_quality = 100 if screenshot_type == 'png' else int(os.getenv("SCREENSHOT_QUALITY", 72))

    while y < min(page_height, SCREENSHOT_MAX_TOTAL_HEIGHT):
        # Only scroll if not at the top (y > 0)
        if y > 0:
            await page.evaluate(f"window.scrollTo(0, {y})")

        # Request GC only before screenshot (not 3x per chunk)
        await page.request_gc()

        screenshot_kwargs = {
            'type': screenshot_type,
            'full_page': False
        }
        # Only pass quality parameter for jpeg (PNG doesn't support it in Playwright)
        if screenshot_type == 'jpeg':
            screenshot_kwargs['quality'] = screenshot_quality

        chunk_start = time.time()
        screenshot_chunks.append(await page.screenshot(**screenshot_kwargs))
        chunk_time = time.time() - chunk_start
        chunk_times.append(chunk_time)
        logger.debug(f"{watch_info}Chunk {len(screenshot_chunks)} captured in {chunk_time:.2f}s")
        y += step_size

    # Restore original viewport size
    await page.set_viewport_size({'width': original_viewport['width'], 'height': original_viewport['height']})

    # Unlock element dimensions if they were locked
    if elements_locked:
        unlock_elements_js_path = os.path.join(os.path.dirname(__file__), 'res', 'unlock-elements-sizing.js')
        with open(unlock_elements_js_path, 'r') as f:
            unlock_elements_js = f.read()
        await page.evaluate(unlock_elements_js)
        logger.debug(f"{watch_info}Element dimensions unlocked after screenshot capture")

    capture_time = time.time() - capture_start
    total_capture_time = sum(chunk_times)
    logger.debug(f"{watch_info}All {len(screenshot_chunks)} chunks captured in {capture_time:.2f}s (total chunk time: {total_capture_time:.2f}s)")

    # If we have multiple chunks, stitch them together
    if len(screenshot_chunks) > 1:
        stitch_start = time.time()
        logger.debug(f"{watch_info}Starting stitching of {len(screenshot_chunks)} chunks")

        # Always use spawn subprocess for ANY stitching (2+ chunks)
        # PIL allocates at C level and Python GC never releases it - subprocess exit forces OS to reclaim
        # Trade-off: 35MB resource_tracker vs 500MB+ PIL leak in main process
        from changedetectionio.content_fetchers.screenshot_handler import stitch_images_worker_raw_bytes
        import multiprocessing
        import struct

        ctx = multiprocessing.get_context('spawn')
        parent_conn, child_conn = ctx.Pipe()
        p = ctx.Process(target=stitch_images_worker_raw_bytes, args=(child_conn, page_height, SCREENSHOT_MAX_TOTAL_HEIGHT))
        p.start()

        # Send via raw bytes (no pickle)
        parent_conn.send_bytes(struct.pack('I', len(screenshot_chunks)))
        for chunk in screenshot_chunks:
            parent_conn.send_bytes(chunk)

        screenshot = parent_conn.recv_bytes()
        p.join()

        parent_conn.close()
        child_conn.close()
        del p, parent_conn, child_conn

        stitch_time = time.time() - stitch_start
        total_time = time.time() - start
        setup_time = total_time - capture_time - stitch_time
        logger.debug(
            f"{watch_info}Screenshot complete - Page height: {page_height}px, Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT}px | "
            f"Setup: {setup_time:.2f}s, Capture: {capture_time:.2f}s, Stitching: {stitch_time:.2f}s, Total: {total_time:.2f}s")
        return screenshot

    total_time = time.time() - start
    setup_time = total_time - capture_time
    logger.debug(
        f"{watch_info}Screenshot complete - Page height: {page_height}px, Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT}px | "
        f"Setup: {setup_time:.2f}s, Single chunk: {capture_time:.2f}s, Total: {total_time:.2f}s")

    return screenshot_chunks[0]

class fetcher(Fetcher):
    fetcher_description = "Playwright {}/Javascript".format(
        os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
    )
    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
        fetcher_description += " via '{}'".format(os.getenv("PLAYWRIGHT_DRIVER_URL"))

    browser_type = ''
    command_executor = ''

    # Configs for Proxy setup
    # In the ENV vars, is prefixed with "playwright_proxy_", so it is for example "playwright_proxy_server"
    playwright_proxy_settings_mappings = ['bypass', 'server', 'username', 'password']

    proxy = None

    # Capability flags
    supports_browser_steps = True
    supports_screenshots = True
    supports_xpath_element_data = True
    supports_request_blocking = True

    # When True, _get_browser() launches a local browser instead of connecting to a remote
    # CDP endpoint (overridden by the html_playwright_builtin subclass).
    local_launch = False

    async def _get_browser(self, browser_type):
        """Acquire a Playwright browser. Default: connect to the configured remote CDP endpoint.
        Subclasses (local launch) override this. Kept as the single seam so the tuned run()
        body is shared."""
        return await browser_type.connect_over_cdp(self.browser_connection_url, timeout=60000)


    def _resolve_browser_type_name(self):
        """chromium/firefox/webkit - from the browser_config when the engine supports choosing
        it (local launch), else the env default."""
        bc = getattr(self, 'browser_config', None)
        if bc is not None and getattr(bc, 'browser_type', None) and self.supports_browser_type:
            return bc.browser_type
        return self.browser_type

    @classmethod
    def get_status_icon_data(cls):
        """Return Chrome browser icon data for Playwright fetcher."""
        return {
            'filename': 'google-chrome-icon.png',
            'alt': 'Using a Chrome browser',
            'title': 'Using a Chrome browser'
        }

    def __init__(self, proxy_override=None, custom_browser_connection_url=None, **kwargs):
        super().__init__(**kwargs)

        self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')

        if custom_browser_connection_url:
            self.browser_connection_is_custom = True
            self.browser_connection_url = custom_browser_connection_url
        else:
            # Fallback to fetching from system
            # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
            self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')

        # If any proxy settings are enabled, then we should setup the proxy object
        proxy_args = {}
        for k in self.playwright_proxy_settings_mappings:
            v = os.getenv('playwright_proxy_' + k, False)
            if v:
                proxy_args[k] = v.strip('"')

        if proxy_args:
            self.proxy = proxy_args

        # allow per-watch proxy selection override
        if proxy_override:
            self.proxy = {'server': proxy_override}

        if self.proxy:
            # Playwright needs separate username and password values
            parsed = urlparse(self.proxy.get('server'))
            if parsed.username:
                self.proxy['username'] = parsed.username
                self.proxy['password'] = parsed.password

    async def screenshot_step(self, step_n=''):
        super().screenshot_step(step_n=step_n)
        watch_uuid = getattr(self, 'watch_uuid', None)
        screenshot = await capture_full_page_async(page=self.page, screenshot_format=self.screenshot_format, watch_uuid=watch_uuid, lock_viewport_elements=self.lock_viewport_elements)

        # Request GC immediately after screenshot to free memory
        # Screenshots can be large and browser steps take many of them
        await self.page.request_gc()

        if self.browser_steps_screenshot_path is not None:
            destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.jpeg'.format(step_n))
            logger.debug(f"Saving step screenshot to {destination}")
            with open(destination, 'wb') as f:
                f.write(screenshot)
            # Clear local reference to allow screenshot bytes to be collected
            del screenshot
            gc.collect()

    async def save_step_html(self, step_n):
        super().save_step_html(step_n=step_n)
        content = await self.page.content()

        # Request GC after getting page content
        await self.page.request_gc()

        destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
        logger.debug(f"Saving step HTML to {destination}")
        with open(destination, 'w', encoding='utf-8') as f:
            f.write(content)
        # Clear local reference
        del content
        gc.collect()

    async def run(self,
                  fetch_favicon=True,
                  current_include_filters=None,
                  empty_pages_are_a_change=False,
                  ignore_status_codes=False,
                  is_binary=False,
                  request_body=None,
                  request_headers=None,
                  request_method=None,
                  screenshot_format=None,
                  timeout=None,
                  url=None,
                  watch_uuid=None,
                  ):

        from playwright.async_api import async_playwright
        import playwright._impl._errors
        import time
        self.delete_browser_steps_screenshots()
        self.watch_uuid = watch_uuid  # Store for use in screenshot_step
        response = None

        async with async_playwright() as p:
            _browser_type_name = self._resolve_browser_type_name()
            browser_type = getattr(p, _browser_type_name)

            # Acquire the browser via the seam (remote connect by default, local launch in the
            # html_playwright_builtin subclass).
            browser = await self._get_browser(browser_type)

            # SOCKS5 with authentication is not supported (yet)
            # https://github.com/microsoft/playwright/issues/10567

            # Per-watch browser behaviour (viewport/locale/timezone) from the resolved
            # FetcherConfig - only added when set so defaults/behaviour are unchanged.
            # (locale also drives the Accept-Language header - see issues #4210 #1412; timezone #3303)
            context_kwargs = dict(
                accept_downloads=False,  # Should never be needed
                # Enabled by default because sites such as GitHub need it for injected JavaScript.
                # Some CDP implementations do not support Page.setBypassCSP, so allow operators to disable it.
                bypass_csp=get_playwright_bypass_csp(),
                extra_http_headers=request_headers,
                ignore_https_errors=True,
                proxy=self.proxy,
                user_agent=manage_user_agent(headers=request_headers),
            )
            # service_workers is a chromium-only new_context option - Firefox/WebKit reject it,
            # so only pass it for chromium (matters for the local html_playwright_builtin fetcher).
            if _browser_type_name == 'chromium':
                context_kwargs['service_workers'] = os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow')
            _bc = getattr(self, 'browser_config', None)
            if _bc is not None:
                context_kwargs.update(_bc.browser_context_kwargs())

            # Set user agent to prevent Cloudflare from blocking the browser
            # Use the default one configured in the App.py model that's passed from fetch_site_status.py
            context = await browser.new_context(**context_kwargs)

            self.page = await context.new_page()

            # Track the LATEST main-frame document response for the whole fetch, not just the one
            # goto() returns. This app compares the text of the page the browser ends up on, and a
            # site that gates with an interstitial (503/429 + meta-refresh) navigates to the real
            # page *during* the extra_wait below - judging the fetch on the first response fails a
            # watch whose content is present and fine. Same for plain client-side redirects.
            # Shared with action_goto_url() so only one 'response' listener exists on the page.
            from changedetectionio.browser_steps.browser_steps import track_latest_navigation_response
            # Must be an identity check - the tracker hands back the same (initially empty, so
            # falsy) dict the listener writes into, and `or {}` would quietly swap in a different
            # one that never gets updated.
            latest_navigation_response = track_latest_navigation_response(self.page)
            if latest_navigation_response is None:
                latest_navigation_response = {}

            # Listen for all console events and handle errors
            self.page.on("console", lambda msg: logger.debug(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))

            # Re-use as much code from browser steps as possible so its the same
            from changedetectionio.browser_steps.browser_steps import steppable_browser_interface
            browsersteps_interface = steppable_browser_interface(start_url=url)
            browsersteps_interface.page = self.page

            response = await browsersteps_interface.action_goto_url(value=url)

            if response is None:
                await context.close()
                await browser.close()
                logger.debug("Content Fetcher > Response object from the browser communication was none")
                raise EmptyReply(url=url, status_code=None)

            # In async_playwright, all_headers() returns a coroutine
            try:
                self.headers = await response.all_headers()
            except TypeError:
                # Fallback for sync version
                self.headers = response.all_headers()

            try:
                if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
                    await browsersteps_interface.action_execute_js(value=self.webdriver_js_execute_code, selector=None)
            except playwright._impl._errors.TimeoutError as e:
                await context.close()
                await browser.close()
                # This can be ok, we will try to grab what we could retrieve
                pass
            except Exception as e:
                logger.debug(f"Content Fetcher > Other exception when executing custom JS code {str(e)}")
                await context.close()
                await browser.close()
                raise PageUnloadable(url=url, status_code=None, message=str(e))

            extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
            await self.page.wait_for_timeout(extra_wait * 1000)

            # A meta-refresh or client-side redirect usually lands during that wait, so judge the
            # fetch on the document we are actually about to extract rather than the first one.
            latest = latest_navigation_response.get('response')
            if latest is not None and latest is not response:
                logger.debug(f"Page navigated again while waiting, judging the fetch on {latest.url} "
                             f"(status {latest.status}) instead of the first response for {url}")
                response = latest
                try:
                    self.headers = await response.all_headers()
                except Exception as e:
                    logger.debug(f"Could not refresh headers from the final document: {e}")

            # Don't extract while a navigation is mid-flight, that is what produces
            # "Execution context was destroyed, most likely because of a navigation"
            try:
                await self.page.wait_for_load_state('load', timeout=extra_wait * 1000)
            except Exception as e:
                logger.debug(f"Page did not reach a settled load state, continuing anyway: {e}")

            try:
                self.status_code = response.status
            except Exception as e:
                # https://github.com/dgtlmoon/changedetection.io/discussions/2122#discussioncomment-8241962
                logger.critical(f"Response from the browser/Playwright did not have a status_code! Response follows.")
                logger.critical(response)
                await context.close()
                await browser.close()
                raise PageUnloadable(url=url, status_code=None, message=str(e))

            if fetch_favicon:
                try:
                    self.favicon_blob = await self.page.evaluate(FAVICON_FETCHER_JS)
                    await self.page.request_gc()
                except Exception as e:
                    logger.error(f"Error fetching FavIcon info {str(e)}, continuing.")

            if self.status_code != 200 and not ignore_status_codes:
                screenshot = await capture_full_page_async(self.page, screenshot_format=self.screenshot_format, watch_uuid=watch_uuid, lock_viewport_elements=self.lock_viewport_elements)
                # Finally block will handle cleanup
                raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)

            if not empty_pages_are_a_change and len((await self.page.content()).strip()) == 0:
                logger.debug("Content Fetcher > Content was empty, empty_pages_are_a_change = False")
                await context.close()
                await browser.close()
                raise EmptyReply(url=url, status_code=response.status)

            # Wrap remaining operations in try/finally to ensure cleanup
            try:
                # Run Browser Steps here
                if self.browser_steps:
                    try:
                        await self.iterate_browser_steps(start_url=url)
                    except BrowserStepsStepException:
                        # Finally block will handle cleanup
                        raise

                    await self.page.wait_for_timeout(extra_wait * 1000)

                now = time.time()
                # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
                if current_include_filters is not None:
                    await self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters)))
                else:
                    await self.page.evaluate("var include_filters=''")
                await self.page.request_gc()

                # request_gc before and after evaluate to free up memory
                # @todo browsersteps etc
                MAX_TOTAL_HEIGHT = int(os.getenv("SCREENSHOT_MAX_HEIGHT", SCREENSHOT_MAX_HEIGHT_DEFAULT))
                self.xpath_data = await self.page.evaluate(XPATH_ELEMENT_JS, {
                    "visualselector_xpath_selectors": visualselector_xpath_selectors,
                    "max_height": MAX_TOTAL_HEIGHT
                })
                await self.page.request_gc()

                self.instock_data = await self.page.evaluate(INSTOCK_DATA_JS)
                await self.page.request_gc()

                self.content = await self.page.content()
                await self.page.request_gc()
                logger.debug(f"Scrape xPath element data in browser done in {time.time() - now:.2f}s")


                # Bug 3 in Playwright screenshot handling
                # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
                # JPEG is better here because the screenshots can be very very large

                # Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
                # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
                # acceptable screenshot quality here
                # The actual screenshot - this always base64 and needs decoding! horrible! huge CPU usage
                self.screenshot = await capture_full_page_async(page=self.page, screenshot_format=self.screenshot_format, watch_uuid=watch_uuid, lock_viewport_elements=self.lock_viewport_elements)

                # Force aggressive memory cleanup - screenshots are large and base64 decode creates temporary buffers
                await self.page.request_gc()
                gc.collect()

            except ScreenshotUnavailable:
                # Re-raise screenshot unavailable exceptions
                raise ScreenshotUnavailable(url=url, status_code=self.status_code)

            finally:
                # Clean up resources properly with timeouts to prevent hanging
                try:
                    if hasattr(self, 'page') and self.page:
                        await self.page.request_gc()
                        await asyncio.wait_for(self.page.close(), timeout=5.0)
                        logger.debug(f"Successfully closed page for {url}")
                except asyncio.TimeoutError:
                    logger.warning(f"Timed out closing page for {url} (5s)")
                except Exception as e:
                    logger.warning(f"Error closing page for {url}: {e}")
                finally:
                    self.page = None

                try:
                    if context:
                        await asyncio.wait_for(context.close(), timeout=5.0)
                        logger.debug(f"Successfully closed context for {url}")
                except asyncio.TimeoutError:
                    logger.warning(f"Timed out closing context for {url} (5s)")
                except Exception as e:
                    logger.warning(f"Error closing context for {url}: {e}")
                finally:
                    context = None

                try:
                    if browser:
                        await asyncio.wait_for(browser.close(), timeout=5.0)
                        logger.debug(f"Successfully closed browser connection for {url}")
                except asyncio.TimeoutError:
                    logger.warning(f"Timed out closing browser connection for {url} (5s)")
                except Exception as e:
                    logger.warning(f"Error closing browser for {url}: {e}")
                finally:
                    browser = None

                # Force Python GC to release Playwright resources immediately
                # Playwright objects can have circular references that delay cleanup
                gc.collect()


# Plugin registration for built-in fetcher
class PlaywrightFetcherPlugin:
    """Plugin class that registers the Playwright fetcher as a built-in plugin."""

    def register_content_fetcher(self):
        """Register the Playwright fetcher"""
        return ('html_webdriver', fetcher)


# Create module-level instance for plugin registration
playwright_plugin = PlaywrightFetcherPlugin()