diff --git a/changedetectionio/content_fetchers/puppeteer.py b/changedetectionio/content_fetchers/puppeteer.py index 78b0dba54..da5e8557c 100644 --- a/changedetectionio/content_fetchers/puppeteer.py +++ b/changedetectionio/content_fetchers/puppeteer.py @@ -459,8 +459,29 @@ class fetcher(Fetcher): logger.debug(f"Content-ready wait of {extra_wait}s elapsed, issuing Page.stopLoading before extracting") await self.page._client.send('Page.stopLoading') logger.debug("stopLoading command sent!") + + # stopLoading stops the network, not script execution. A page whose JS has pegged + # the renderer's main thread (a runaway loop, a rAF that never settles) holds that + # thread indefinitely, and every CDP call that needs to run script then queues + # behind it and never returns - page.content, the xPath scraper, the favicon + # fetcher. The fetch dies at PUPPETEER_MAX_PROCESSING_TIMEOUT_SECONDS having + # extracted nothing, with a core spinning the entire time. + # + # Nothing else recovers this. Runtime.evaluate's own `timeout` parameter bounds an + # evaluation once it starts, not time spent queued behind the running task, and + # wrapping the call in asyncio.wait_for is worse than useless: cancelling a + # pyppeteer request mid-flight leaves the connection unusable ("Target closed" on + # everything after it). Terminating execution is what releases the thread - + # measured against a deliberately spinning page, extraction went from timing out + # to returning the full DOM in 0.0s and the renderer dropped from 1.00 to 0.08 + # cores. Safe here because stopLoading has already declared "give me what + # rendered", and the content-ready wait above has already had its chance to let + # late JS-rendered content appear. + await self.page._client.send('Runtime.terminateExecution') + logger.debug("Runtime.terminateExecution sent, any runaway page script is stopped") except Exception as e: - logger.debug(f"Page.stopLoading skipped, page is most likely already gone: {e}") + logger.debug(f"Page.stopLoading/Runtime.terminateExecution skipped, page is most " + f"likely already gone: {e}") # Track the LATEST main-frame document response for the whole fetch, not just the one that # goto() happens to return. This app compares the text of the page the browser ends up on,