From bb729894bc8bcfa2af007aca26d02698d600333c Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sat, 12 Sep 2026 10:49:11 +0200 Subject: [PATCH] Browser fetchers - Report the real status code when Chrome aborts a bodiless error response Chrome 153+ refuses to commit a navigation when a 4xx/5xx arrives with a zero-length body: page.goto() raises net::ERR_HTTP_RESPONSE_CODE_FAILURE instead of returning the response. The response is received fine, we just never get it as a return value, so the raw net:: string landed in last_error instead of "Error - 404". Verified against two browser images, same HTTP server: Chrome 153 empty-body 404 -> raises ERR_HTTP_RESPONSE_CODE_FAILURE Chrome 153 404 with body -> status=404 Chromium 119 empty-body 404 -> status=404 Chromium 119 404 with body -> status=404 The fix keeps the main-frame response from the 'response' event and hands that back when goto raises, so .status / .all_headers() and the existing non-200 branch (which also captures the screenshot) work unchanged. The latest matching response wins, so a redirect chain still reports its final hop. Any other error re-raises as before, and if no response was captured we re-raise too - the status is never invented, which keeps older browsers on exactly their old path. Two independent navigation sites needed it: - browser_steps.py action_goto_url - covers the playwright fetcher, the live Browser Steps UI, the Goto URL / Goto site steps, and the CloakBrowser plugin which imports it. This is the one that broke CI: content_fetchers/__init__.py forces playwright when a watch has browser steps, so test_non_200_errors_report_browsersteps ran the playwright path in the pyppeteer jobs too. - puppeteer.py - its own goto retry loop, used when FAST_PUPPETEER_CHROME_FETCHER is set and the watch has no browser steps. No test covers that path; verified by driving the fetcher directly. Note pyppeteer exposes isNavigationRequest / frame / mainFrame as properties where playwright uses is_navigation_request() as a method. Mixing them up raises 'bool' object is not callable, which gets swallowed as a renderer page error rather than failing loudly. Checked against the pinned pyppeteer-ng==2.0.0rc16. Selenium is unaffected - it hardcodes status_code = 200 because WebDriver cannot see the HTTP status, so it never reaches the non-200 branch. Tested with the full CI browser set (test_content, test_errorhandling, test_fetch_data, test_custom_js_before_content): 10 passed on each of Chrome 153 + playwright, Chrome 153 + pyppeteer, Chromium 119 + playwright, Chromium 119 + pyppeteer, the last two against a canonical Dockerfile.chromium119 build so Chromium is the only variable. Co-Authored-By: Claude Opus 5 (1M context) --- .../browser_steps/browser_steps.py | 26 ++++++++- .../content_fetchers/puppeteer.py | 57 +++++++++++++------ 2 files changed, 66 insertions(+), 17 deletions(-) diff --git a/changedetectionio/browser_steps/browser_steps.py b/changedetectionio/browser_steps/browser_steps.py index 3aace66dc..fdb587412 100644 --- a/changedetectionio/browser_steps/browser_steps.py +++ b/changedetectionio/browser_steps/browser_steps.py @@ -145,8 +145,32 @@ class steppable_browser_interface(): # and private-IP SSRF possible via a browser step (GHSA-hm22-wg2m-35v4). await validate_fetch_url_async(value) + # Chrome 153+ refuses to commit a navigation when an error status arrives with a + # zero-length body, so page.goto() raises net::ERR_HTTP_RESPONSE_CODE_FAILURE instead of + # handing back the response. The response was received fine, we just never get it as a + # return value, so keep the main-frame response from the 'response' event and hand that + # back - callers then report a real "Error - 404" instead of a raw net:: string. + # Kept as the latest matching response so a redirect chain reports its final hop. + navigation_response = {} + + def _keep_navigation_response(response): + if response.frame == self.page.main_frame and response.request.is_navigation_request(): + navigation_response['response'] = response + + self.page.on("response", _keep_navigation_response) + now = time.time() - response = await self.page.goto(value, timeout=0, wait_until='load') + try: + response = await self.page.goto(value, timeout=0, wait_until='load') + except Exception as e: + if 'ERR_HTTP_RESPONSE_CODE_FAILURE' not in str(e) or not navigation_response: + raise + response = navigation_response['response'] + logger.debug(f"Navigation was aborted by the browser (empty body on an error status), " + f"recovered status {response.status} from the response event") + finally: + self.page.remove_listener("response", _keep_navigation_response) + logger.debug(f"Time to goto URL {time.time()-now:.2f}s") return response diff --git a/changedetectionio/content_fetchers/puppeteer.py b/changedetectionio/content_fetchers/puppeteer.py index 362e0d167..6e9acbb1a 100644 --- a/changedetectionio/content_fetchers/puppeteer.py +++ b/changedetectionio/content_fetchers/puppeteer.py @@ -430,25 +430,50 @@ class fetcher(Fetcher): # Listen for first response to trigger frame handler setup self.page._client.on('Network.responseReceived', setup_frame_handlers_on_first_response) + # Chrome 153+ refuses to commit a navigation when an error status arrives with a + # zero-length body, so goto() raises net::ERR_HTTP_RESPONSE_CODE_FAILURE instead of handing + # back the response. The response was received fine, we just never get it as a return value, + # so keep the main-frame response from the 'response' event and use that instead - the + # status check below then reports a real "Error - 404" instead of a raw net:: string. + # Kept as the latest matching response so a redirect chain reports its final hop. + navigation_response = {} + + def _keep_navigation_response(response): + # Note pyppeteer exposes these as properties, unlike playwright where they are methods + if response.frame == self.page.mainFrame and response.request.isNavigationRequest: + navigation_response['response'] = response + + self.page.on('response', _keep_navigation_response) + response = None attempt=0 - while not response: - logger.debug(f"Attempting page fetch {url} attempt {attempt}") - asyncio.create_task(handle_frame_navigation()) - response = await self.page.goto(url, timeout=0) - await asyncio.sleep(1 + extra_wait) - # Check if page still exists before sending command - if self.page and hasattr(self.page, '_client'): - await self.page._client.send('Page.stopLoading') + try: + while not response: + logger.debug(f"Attempting page fetch {url} attempt {attempt}") + asyncio.create_task(handle_frame_navigation()) + try: + response = await self.page.goto(url, timeout=0) + except Exception as e: + if 'ERR_HTTP_RESPONSE_CODE_FAILURE' not in str(e) or not navigation_response: + raise + response = navigation_response['response'] + logger.debug(f"Navigation was aborted by the browser (empty body on an error status), " + f"recovered status {response.status} from the response event") + await asyncio.sleep(1 + extra_wait) + # Check if page still exists before sending command + if self.page and hasattr(self.page, '_client'): + await self.page._client.send('Page.stopLoading') - if response: - break - if not response: - logger.warning("Page did not fetch! trying again!") - if response is None and attempt>=2: - logger.warning(f"Content Fetcher > Response object was none (as in, the response from the browser was empty, not just the content) exiting attempt {attempt}") - raise EmptyReply(url=url, status_code=None) - attempt+=1 + if response: + break + if not response: + logger.warning("Page did not fetch! trying again!") + if response is None and attempt>=2: + logger.warning(f"Content Fetcher > Response object was none (as in, the response from the browser was empty, not just the content) exiting attempt {attempt}") + raise EmptyReply(url=url, status_code=None) + attempt+=1 + finally: + self.page.remove_listener('response', _keep_navigation_response) self.headers = response.headers