fix(linux): prevent X11 BadWindow crash in get_focused_display (#14561)

* fix(linux): prevent X11 BadWindow crash in get_focused_display

When the active window is destroyed between xdo_get_active_window and
xdo_get_window_location/xdo_get_window_size calls, the default X11
error handler terminates the process with a BadWindow error. This
causes the rustdesk --server process to crash and the remote session
to disconnect and reconnect every time the user closes a window.

Install a custom X error handler around the xdo calls that catches
BadWindow errors and returns gracefully instead of crashing.

Fixes: https://github.com/rustdesk/rustdesk/issues/9003

Co-Authored-By: Claude (claude-opus-4-6) <noreply@anthropic.com>
Signed-off-by: easonysliu <easonysliu@tencent.com>

* fix(linux): prevent BadWindow crash in focus display lookup

Signed-off-by: fufesou <linlong1266@gmail.com>

---------

Signed-off-by: easonysliu <easonysliu@tencent.com>
Signed-off-by: fufesou <linlong1266@gmail.com>
Co-authored-by: easonysliu <easonysliu@tencent.com>
Co-authored-by: Claude (claude-opus-4-6) <noreply@anthropic.com>
Co-authored-by: fufesou <linlong1266@gmail.com>

src/platform/linux.rs

@@ -6,7 +6,7 @@ use hbb_common::{
     anyhow::anyhow,
     bail,
     config::{keys::OPTION_ALLOW_LINUX_HEADLESS, Config},
-    libc::{c_char, c_int, c_long, c_uint, c_void},
+    libc::{c_char, c_int, c_long, c_uint, c_ulong, c_void},
     log,
     message_proto::{DisplayInfo, Resolution},
     regex::{Captures, Regex},
@@ -97,10 +97,55 @@ thread_local! {
     static DISPLAY: RefCell<*mut c_void> = RefCell::new(unsafe { XOpenDisplay(std::ptr::null())});
 }
 
+// X11 error event structure for the custom error handler.
+// See: https://www.x.org/releases/current/doc/libX11/libX11/libX11.html#Using-the-Default-Error-Handlers
+#[repr(C)]
+struct XErrorEvent {
+    type_: c_int,
+    display: *mut c_void, // Display*
+    resourceid: c_ulong,  // XID
+    serial: c_ulong,
+    error_code: u8,
+    request_code: u8,
+    minor_code: u8,
+}
+
+type XErrorHandler = unsafe extern "C" fn(*mut c_void, *mut XErrorEvent) -> c_int;
+
+const X11_BAD_WINDOW: u8 = 3;
+const XDO_SUCCESS: c_int = 0;
+const XDO_ERROR: c_int = 1;
+
+/// Atomic flag set by the custom X error handler when a BadWindow error occurs.
+static X_BAD_WINDOW_DETECTED: AtomicBool = AtomicBool::new(false);
+static X_UNEXPECTED_ERROR_DETECTED: AtomicBool = AtomicBool::new(false);
+
+/// Custom X error handler that catches BadWindow errors (error_code == 3) instead of
+/// letting the default handler terminate the process.
+/// See issue: https://github.com/rustdesk/rustdesk/issues/9003
+unsafe extern "C" fn handle_x_error(_display: *mut c_void, event: *mut XErrorEvent) -> c_int {
+    if !event.is_null() && (*event).error_code == X11_BAD_WINDOW {
+        X_BAD_WINDOW_DETECTED.store(true, Ordering::SeqCst);
+        log::debug!("Caught X11 BadWindow error (suppressed), window was likely destroyed");
+        return 0;
+    }
+    X_UNEXPECTED_ERROR_DETECTED.store(true, Ordering::SeqCst);
+    if !event.is_null() {
+        log::warn!(
+            "X11 error: error_code={}, request_code={}, minor_code={}",
+            (*event).error_code,
+            (*event).request_code,
+            (*event).minor_code,
+        );
+    }
+    0
+}
+
 #[link(name = "X11")]
 extern "C" {
     fn XOpenDisplay(display_name: *const c_char) -> *mut c_void;
     // fn XCloseDisplay(d: *mut c_void) -> c_int;
+    fn XSetErrorHandler(handler: Option<XErrorHandler>) -> Option<XErrorHandler>;
 }
 
 #[link(name = "Xfixes")]
@@ -231,25 +276,47 @@ pub fn get_focused_display(displays: Vec<DisplayInfo>) -> Option<usize> {
                 if libxdo_sys::xdo_get_active_window(*xdo as *const _, &mut window) != 0 {
                     return;
                 }
-                if libxdo_sys::xdo_get_window_location(
+
+                // XSetErrorHandler is process-global, not scoped to this Display/thread.
+                // This path is currently called by the single window_focus service thread.
+                // While installed, this handler can still observe unrelated X11 errors from
+                // other threads; unexpected errors make this geometry query fail.
+                X_BAD_WINDOW_DETECTED.store(false, Ordering::SeqCst);
+                X_UNEXPECTED_ERROR_DETECTED.store(false, Ordering::SeqCst);
+                let prev_handler = XSetErrorHandler(Some(handle_x_error));
+
+                let loc_ret = libxdo_sys::xdo_get_window_location(
                     *xdo as *const _,
                     window,
                     &mut x as _,
                     &mut y as _,
                     std::ptr::null_mut(),
-                ) != 0
+                );
-                {
+                let size_ret = if loc_ret == XDO_SUCCESS {
-                    return;
+                    libxdo_sys::xdo_get_window_size(
-                }
+                        *xdo as *const _,
-                if libxdo_sys::xdo_get_window_size(
+                        window,
-                    *xdo as *const _,
+                        &mut width,
-                    window,
+                        &mut height,
-                    &mut width,
+                    )
-                    &mut height,
+                } else {
-                ) != 0
+                    XDO_ERROR
+                };
+
+                // Do not call XSync(DISPLAY) here: DISPLAY is a separate
+                // XOpenDisplay() connection, while libxdo owns the Display*
+                // used by these geometry queries. These libxdo calls are
+                // synchronous XGetWindowAttributes-based queries, so the target
+                // BadWindow is expected to be delivered before the calls return.
+                XSetErrorHandler(prev_handler);
+                if X_BAD_WINDOW_DETECTED.load(Ordering::SeqCst)
+                    || X_UNEXPECTED_ERROR_DETECTED.load(Ordering::SeqCst)
+                    || loc_ret != XDO_SUCCESS
+                    || size_ret != XDO_SUCCESS
                 {
                     return;
                 }
+
                 let center_x = x + (width / 2) as c_int;
                 let center_y = y + (height / 2) as c_int;
                 res = displays.iter().position(|d| {
@@ -2150,7 +2217,10 @@ pub fn clear_gnome_shortcuts_inhibitor_permission() -> ResultType<()> {
                 || err_name == "org.freedesktop.DBus.Error.UnknownObject"
                 || err_name == "org.freedesktop.DBus.Error.ServiceUnknown"
             {
-                log::info!("GNOME shortcuts inhibitor permission was not set ({})", err_name);
+                log::info!(
+                    "GNOME shortcuts inhibitor permission was not set ({})",
+                    err_name
+                );
                 Ok(())
             } else {
                 bail!("Failed to clear permission: {}", e)