From 7a29b5bc731c2c7e9385b5debbbf68bd93c4aaaf Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 14 Sep 2026 12:40:53 +0200 Subject: [PATCH] Page fetching - Terminate runaway page script before extracting, or a spinning renderer eats the whole fetch (#4433) Page.stopLoading stops the network, not script execution. A page whose JavaScript has pegged the renderer's main thread keeps that thread indefinitely, and every CDP call that needs to run script then queues behind it and never returns - page.content, the xPath scraper, the favicon fetcher. The fetch dies at PUPPETEER_MAX_PROCESSING_TIMEOUT_SECONDS having extracted nothing, with a core spinning the entire time. Seen in production on a watch that failed every check for days: the renderer sat at 1.04-1.07 cores for the full 60s budget (sampled every 2s, flat), 33s of which was a single unanswered Runtime.evaluate, and the watch logged "xpath_data length returned empty" every time. Nothing else recovers this state. Runtime.evaluate's own `timeout` parameter bounds an evaluation once it starts, not time spent queued behind the running task - measured, it still hung past 15s. Wrapping the call in asyncio.wait_for is worse than useless: cancelling a pyppeteer request mid-flight leaves the connection unusable, with "Protocol error: Target closed" on everything after it. Runtime.terminateExecution is what releases the thread. Against a page that fires load and then spins forever, through the real fetcher: before: 60.6s, BrowserFetchTimedOut, 0 bytes content, no xpath_data, no screenshot after: 5.6s, no exception, content + payload, xpath_data present, screenshot 8415b and the renderer drops from 1.00 to 0.08 cores. Safe at this point in the fetch: stopLoading has already declared "give me what rendered", and the content-ready wait above has already had its chance to let late JS-rendered content appear. Co-authored-by: Claude Opus 5 (1M context) --- .../content_fetchers/puppeteer.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/changedetectionio/content_fetchers/puppeteer.py b/changedetectionio/content_fetchers/puppeteer.py index 78b0dba54..da5e8557c 100644 --- a/changedetectionio/content_fetchers/puppeteer.py +++ b/changedetectionio/content_fetchers/puppeteer.py @@ -459,8 +459,29 @@ class fetcher(Fetcher): logger.debug(f"Content-ready wait of {extra_wait}s elapsed, issuing Page.stopLoading before extracting") await self.page._client.send('Page.stopLoading') logger.debug("stopLoading command sent!") + + # stopLoading stops the network, not script execution. A page whose JS has pegged + # the renderer's main thread (a runaway loop, a rAF that never settles) holds that + # thread indefinitely, and every CDP call that needs to run script then queues + # behind it and never returns - page.content, the xPath scraper, the favicon + # fetcher. The fetch dies at PUPPETEER_MAX_PROCESSING_TIMEOUT_SECONDS having + # extracted nothing, with a core spinning the entire time. + # + # Nothing else recovers this. Runtime.evaluate's own `timeout` parameter bounds an + # evaluation once it starts, not time spent queued behind the running task, and + # wrapping the call in asyncio.wait_for is worse than useless: cancelling a + # pyppeteer request mid-flight leaves the connection unusable ("Target closed" on + # everything after it). Terminating execution is what releases the thread - + # measured against a deliberately spinning page, extraction went from timing out + # to returning the full DOM in 0.0s and the renderer dropped from 1.00 to 0.08 + # cores. Safe here because stopLoading has already declared "give me what + # rendered", and the content-ready wait above has already had its chance to let + # late JS-rendered content appear. + await self.page._client.send('Runtime.terminateExecution') + logger.debug("Runtime.terminateExecution sent, any runaway page script is stopped") except Exception as e: - logger.debug(f"Page.stopLoading skipped, page is most likely already gone: {e}") + logger.debug(f"Page.stopLoading/Runtime.terminateExecution skipped, page is most " + f"likely already gone: {e}") # Track the LATEST main-frame document response for the whole fetch, not just the one that # goto() happens to return. This app compares the text of the page the browser ends up on,