From ee8cc0c06b86430ad274fdc36fbfded6ae8fb5ef Mon Sep 17 00:00:00 2001 From: eason <85663565+mango766@users.noreply.github.com> Date: Tue, 28 Apr 2026 11:04:29 +0800 Subject: [PATCH] fix(linux): prevent X11 BadWindow crash in get_focused_display (#14561) * fix(linux): prevent X11 BadWindow crash in get_focused_display When the active window is destroyed between xdo_get_active_window and xdo_get_window_location/xdo_get_window_size calls, the default X11 error handler terminates the process with a BadWindow error. This causes the rustdesk --server process to crash and the remote session to disconnect and reconnect every time the user closes a window. Install a custom X error handler around the xdo calls that catches BadWindow errors and returns gracefully instead of crashing. Fixes: https://github.com/rustdesk/rustdesk/issues/9003 Co-Authored-By: Claude (claude-opus-4-6) Signed-off-by: easonysliu * fix(linux): prevent BadWindow crash in focus display lookup Signed-off-by: fufesou --------- Signed-off-by: easonysliu Signed-off-by: fufesou Co-authored-by: easonysliu Co-authored-by: Claude (claude-opus-4-6) Co-authored-by: fufesou --- src/platform/linux.rs | 96 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 83 insertions(+), 13 deletions(-) diff --git a/src/platform/linux.rs b/src/platform/linux.rs index 9493e1cae..7157da760 100644 --- a/src/platform/linux.rs +++ b/src/platform/linux.rs @@ -6,7 +6,7 @@ use hbb_common::{ anyhow::anyhow, bail, config::{keys::OPTION_ALLOW_LINUX_HEADLESS, Config}, - libc::{c_char, c_int, c_long, c_uint, c_void}, + libc::{c_char, c_int, c_long, c_uint, c_ulong, c_void}, log, message_proto::{DisplayInfo, Resolution}, regex::{Captures, Regex}, @@ -97,10 +97,55 @@ thread_local! { static DISPLAY: RefCell<*mut c_void> = RefCell::new(unsafe { XOpenDisplay(std::ptr::null())}); } +// X11 error event structure for the custom error handler. +// See: https://www.x.org/releases/current/doc/libX11/libX11/libX11.html#Using-the-Default-Error-Handlers +#[repr(C)] +struct XErrorEvent { + type_: c_int, + display: *mut c_void, // Display* + resourceid: c_ulong, // XID + serial: c_ulong, + error_code: u8, + request_code: u8, + minor_code: u8, +} + +type XErrorHandler = unsafe extern "C" fn(*mut c_void, *mut XErrorEvent) -> c_int; + +const X11_BAD_WINDOW: u8 = 3; +const XDO_SUCCESS: c_int = 0; +const XDO_ERROR: c_int = 1; + +/// Atomic flag set by the custom X error handler when a BadWindow error occurs. +static X_BAD_WINDOW_DETECTED: AtomicBool = AtomicBool::new(false); +static X_UNEXPECTED_ERROR_DETECTED: AtomicBool = AtomicBool::new(false); + +/// Custom X error handler that catches BadWindow errors (error_code == 3) instead of +/// letting the default handler terminate the process. +/// See issue: https://github.com/rustdesk/rustdesk/issues/9003 +unsafe extern "C" fn handle_x_error(_display: *mut c_void, event: *mut XErrorEvent) -> c_int { + if !event.is_null() && (*event).error_code == X11_BAD_WINDOW { + X_BAD_WINDOW_DETECTED.store(true, Ordering::SeqCst); + log::debug!("Caught X11 BadWindow error (suppressed), window was likely destroyed"); + return 0; + } + X_UNEXPECTED_ERROR_DETECTED.store(true, Ordering::SeqCst); + if !event.is_null() { + log::warn!( + "X11 error: error_code={}, request_code={}, minor_code={}", + (*event).error_code, + (*event).request_code, + (*event).minor_code, + ); + } + 0 +} + #[link(name = "X11")] extern "C" { fn XOpenDisplay(display_name: *const c_char) -> *mut c_void; // fn XCloseDisplay(d: *mut c_void) -> c_int; + fn XSetErrorHandler(handler: Option) -> Option; } #[link(name = "Xfixes")] @@ -231,25 +276,47 @@ pub fn get_focused_display(displays: Vec) -> Option { if libxdo_sys::xdo_get_active_window(*xdo as *const _, &mut window) != 0 { return; } - if libxdo_sys::xdo_get_window_location( + + // XSetErrorHandler is process-global, not scoped to this Display/thread. + // This path is currently called by the single window_focus service thread. + // While installed, this handler can still observe unrelated X11 errors from + // other threads; unexpected errors make this geometry query fail. + X_BAD_WINDOW_DETECTED.store(false, Ordering::SeqCst); + X_UNEXPECTED_ERROR_DETECTED.store(false, Ordering::SeqCst); + let prev_handler = XSetErrorHandler(Some(handle_x_error)); + + let loc_ret = libxdo_sys::xdo_get_window_location( *xdo as *const _, window, &mut x as _, &mut y as _, std::ptr::null_mut(), - ) != 0 - { - return; - } - if libxdo_sys::xdo_get_window_size( - *xdo as *const _, - window, - &mut width, - &mut height, - ) != 0 + ); + let size_ret = if loc_ret == XDO_SUCCESS { + libxdo_sys::xdo_get_window_size( + *xdo as *const _, + window, + &mut width, + &mut height, + ) + } else { + XDO_ERROR + }; + + // Do not call XSync(DISPLAY) here: DISPLAY is a separate + // XOpenDisplay() connection, while libxdo owns the Display* + // used by these geometry queries. These libxdo calls are + // synchronous XGetWindowAttributes-based queries, so the target + // BadWindow is expected to be delivered before the calls return. + XSetErrorHandler(prev_handler); + if X_BAD_WINDOW_DETECTED.load(Ordering::SeqCst) + || X_UNEXPECTED_ERROR_DETECTED.load(Ordering::SeqCst) + || loc_ret != XDO_SUCCESS + || size_ret != XDO_SUCCESS { return; } + let center_x = x + (width / 2) as c_int; let center_y = y + (height / 2) as c_int; res = displays.iter().position(|d| { @@ -2150,7 +2217,10 @@ pub fn clear_gnome_shortcuts_inhibitor_permission() -> ResultType<()> { || err_name == "org.freedesktop.DBus.Error.UnknownObject" || err_name == "org.freedesktop.DBus.Error.ServiceUnknown" { - log::info!("GNOME shortcuts inhibitor permission was not set ({})", err_name); + log::info!( + "GNOME shortcuts inhibitor permission was not set ({})", + err_name + ); Ok(()) } else { bail!("Failed to clear permission: {}", e)