feat(wayland): keyboard mode, legacy translate (#14317)

Signed-off-by: fufesou <linlong1266@gmail.com>

src/server/rdp_input.rs

@@ -1,7 +1,8 @@
-use crate::uinput::service::map_key;
+use super::input_service::set_clipboard_for_paste_sync;
+use crate::uinput::service::{can_input_via_keysym, char_to_keysym, map_key};
 use dbus::{blocking::SyncConnection, Path};
 use enigo::{Key, KeyboardControllable, MouseButton, MouseControllable};
-use hbb_common::ResultType;
+use hbb_common::{log, ResultType};
 use scrap::wayland::pipewire::{get_portal, PwStreamInfo};
 use scrap::wayland::remote_desktop_portal::OrgFreedesktopPortalRemoteDesktop as remote_desktop_portal;
 use std::collections::HashMap;
@@ -19,14 +20,74 @@ pub mod client {
     const PRESSED_DOWN_STATE: u32 = 1;
     const PRESSED_UP_STATE: u32 = 0;
 
+    /// Modifier key state tracking for RDP input.
+    /// Portal API doesn't provide a way to query key state, so we track it ourselves.
+    #[derive(Default)]
+    struct ModifierState {
+        shift_left: bool,
+        shift_right: bool,
+        ctrl_left: bool,
+        ctrl_right: bool,
+        alt_left: bool,
+        alt_right: bool,
+        meta_left: bool,
+        meta_right: bool,
+    }
+
+    impl ModifierState {
+        fn update(&mut self, key: &Key, down: bool) {
+            match key {
+                Key::Shift => self.shift_left = down,
+                Key::RightShift => self.shift_right = down,
+                Key::Control => self.ctrl_left = down,
+                Key::RightControl => self.ctrl_right = down,
+                Key::Alt => self.alt_left = down,
+                Key::RightAlt => self.alt_right = down,
+                Key::Meta | Key::Super | Key::Windows | Key::Command => self.meta_left = down,
+                Key::RWin => self.meta_right = down,
+                // Handle raw keycodes for modifier keys (Linux evdev codes + 8)
+                // In translate mode, modifier keys may be sent as Chr events with raw keycodes.
+                // The +8 offset converts evdev codes to X11/XKB keycodes.
+                Key::Raw(code) => {
+                    const EVDEV_OFFSET: u16 = 8;
+                    const KEY_LEFTSHIFT: u16 = evdev::Key::KEY_LEFTSHIFT.code() + EVDEV_OFFSET;
+                    const KEY_RIGHTSHIFT: u16 = evdev::Key::KEY_RIGHTSHIFT.code() + EVDEV_OFFSET;
+                    const KEY_LEFTCTRL: u16 = evdev::Key::KEY_LEFTCTRL.code() + EVDEV_OFFSET;
+                    const KEY_RIGHTCTRL: u16 = evdev::Key::KEY_RIGHTCTRL.code() + EVDEV_OFFSET;
+                    const KEY_LEFTALT: u16 = evdev::Key::KEY_LEFTALT.code() + EVDEV_OFFSET;
+                    const KEY_RIGHTALT: u16 = evdev::Key::KEY_RIGHTALT.code() + EVDEV_OFFSET;
+                    const KEY_LEFTMETA: u16 = evdev::Key::KEY_LEFTMETA.code() + EVDEV_OFFSET;
+                    const KEY_RIGHTMETA: u16 = evdev::Key::KEY_RIGHTMETA.code() + EVDEV_OFFSET;
+                    match *code {
+                        KEY_LEFTSHIFT => self.shift_left = down,
+                        KEY_RIGHTSHIFT => self.shift_right = down,
+                        KEY_LEFTCTRL => self.ctrl_left = down,
+                        KEY_RIGHTCTRL => self.ctrl_right = down,
+                        KEY_LEFTALT => self.alt_left = down,
+                        KEY_RIGHTALT => self.alt_right = down,
+                        KEY_LEFTMETA => self.meta_left = down,
+                        KEY_RIGHTMETA => self.meta_right = down,
+                        _ => {}
+                    }
+                }
+                _ => {}
+            }
+        }
+    }
+
     pub struct RdpInputKeyboard {
         conn: Arc<SyncConnection>,
         session: Path<'static>,
+        modifier_state: ModifierState,
     }
 
     impl RdpInputKeyboard {
         pub fn new(conn: Arc<SyncConnection>, session: Path<'static>) -> ResultType<Self> {
-            Ok(Self { conn, session })
+            Ok(Self {
+                conn,
+                session,
+                modifier_state: ModifierState::default(),
+            })
         }
     }
 
@@ -39,29 +100,192 @@ pub mod client {
             self
         }
 
-        fn get_key_state(&mut self, _: Key) -> bool {
-            // no api for this
-            false
+        fn get_key_state(&mut self, key: Key) -> bool {
+            // Use tracked modifier state for supported keys
+            match key {
+                Key::Shift => self.modifier_state.shift_left,
+                Key::RightShift => self.modifier_state.shift_right,
+                Key::Control => self.modifier_state.ctrl_left,
+                Key::RightControl => self.modifier_state.ctrl_right,
+                Key::Alt => self.modifier_state.alt_left,
+                Key::RightAlt => self.modifier_state.alt_right,
+                Key::Meta | Key::Super | Key::Windows | Key::Command => {
+                    self.modifier_state.meta_left
+                }
+                Key::RWin => self.modifier_state.meta_right,
+                _ => false,
+            }
         }
 
         fn key_sequence(&mut self, s: &str) {
             for c in s.chars() {
-                let key = Key::Layout(c);
-                let _ = handle_key(true, key, self.conn.clone(), &self.session);
-                let _ = handle_key(false, key, self.conn.clone(), &self.session);
+                let keysym = char_to_keysym(c);
+                // ASCII characters: use keysym
+                if can_input_via_keysym(c, keysym) {
+                    if let Err(e) = send_keysym(keysym, true, self.conn.clone(), &self.session) {
+                        log::error!("Failed to send keysym down: {:?}", e);
+                    }
+                    if let Err(e) = send_keysym(keysym, false, self.conn.clone(), &self.session) {
+                        log::error!("Failed to send keysym up: {:?}", e);
+                    }
+                } else {
+                    // Non-ASCII: use clipboard
+                    input_text_via_clipboard(&c.to_string(), self.conn.clone(), &self.session);
+                }
             }
         }
 
         fn key_down(&mut self, key: Key) -> enigo::ResultType {
-            handle_key(true, key, self.conn.clone(), &self.session)?;
+            if let Key::Layout(chr) = key {
+                let keysym = char_to_keysym(chr);
+                // ASCII characters: use keysym
+                if can_input_via_keysym(chr, keysym) {
+                    send_keysym(keysym, true, self.conn.clone(), &self.session)?;
+                } else {
+                    // Non-ASCII: use clipboard (complete key press in key_down)
+                    input_text_via_clipboard(&chr.to_string(), self.conn.clone(), &self.session);
+                }
+            } else {
+                handle_key(true, key.clone(), self.conn.clone(), &self.session)?;
+                // Update modifier state only after successful send —
+                // if handle_key fails, we don't want stale "pressed" state
+                // affecting subsequent key event decisions.
+                self.modifier_state.update(&key, true);
+            }
             Ok(())
         }
+
         fn key_up(&mut self, key: Key) {
-            let _ = handle_key(false, key, self.conn.clone(), &self.session);
+            // Intentionally asymmetric with key_down: update state BEFORE sending.
+            // On release, we always mark as released even if the send fails below,
+            // to avoid permanently stuck-modifier state in our tracker. The trade-off
+            // (tracker says "released" while OS may still have it pressed) is acceptable
+            // because such failures are rare and subsequent events will resynchronize.
+            self.modifier_state.update(&key, false);
+
+            if let Key::Layout(chr) = key {
+                // ASCII characters: send keysym up if we also sent it on key_down
+                let keysym = char_to_keysym(chr);
+                if can_input_via_keysym(chr, keysym) {
+                    if let Err(e) = send_keysym(keysym, false, self.conn.clone(), &self.session)
+                    {
+                        log::error!("Failed to send keysym up: {:?}", e);
+                    }
+                }
+                // Non-ASCII: already handled completely in key_down via clipboard paste,
+                // no corresponding release needed (clipboard paste is an atomic operation)
+            } else {
+                if let Err(e) = handle_key(false, key, self.conn.clone(), &self.session) {
+                    log::error!("Failed to handle key up: {:?}", e);
+                }
+            }
         }
+
         fn key_click(&mut self, key: Key) {
-            let _ = handle_key(true, key, self.conn.clone(), &self.session);
-            let _ = handle_key(false, key, self.conn.clone(), &self.session);
+            if let Key::Layout(chr) = key {
+                let keysym = char_to_keysym(chr);
+                // ASCII characters: use keysym
+                if can_input_via_keysym(chr, keysym) {
+                    if let Err(e) = send_keysym(keysym, true, self.conn.clone(), &self.session) {
+                        log::error!("Failed to send keysym down: {:?}", e);
+                    }
+                    if let Err(e) = send_keysym(keysym, false, self.conn.clone(), &self.session) {
+                        log::error!("Failed to send keysym up: {:?}", e);
+                    }
+                } else {
+                    // Non-ASCII: use clipboard
+                    input_text_via_clipboard(&chr.to_string(), self.conn.clone(), &self.session);
+                }
+            } else {
+                if let Err(e) = handle_key(true, key.clone(), self.conn.clone(), &self.session) {
+                    log::error!("Failed to handle key down: {:?}", e);
+                } else {
+                    // Only mark modifier as pressed if key-down was actually delivered
+                    self.modifier_state.update(&key, true);
+                }
+                // Always mark as released to avoid stuck-modifier state
+                self.modifier_state.update(&key, false);
+                if let Err(e) = handle_key(false, key, self.conn.clone(), &self.session) {
+                    log::error!("Failed to handle key up: {:?}", e);
+                }
+            }
+        }
+    }
+
+    /// Input text via clipboard + Shift+Insert.
+    /// Shift+Insert is more universal than Ctrl+V, works in both GUI apps and terminals.
+    ///
+    /// Note: Clipboard content is NOT restored after paste - see `set_clipboard_for_paste_sync` for rationale.
+    fn input_text_via_clipboard(text: &str, conn: Arc<SyncConnection>, session: &Path<'static>) {
+        if text.is_empty() {
+            return;
+        }
+        if !set_clipboard_for_paste_sync(text) {
+            return;
+        }
+
+        let portal = get_portal(&conn);
+        let shift_keycode = evdev::Key::KEY_LEFTSHIFT.code() as i32;
+        let insert_keycode = evdev::Key::KEY_INSERT.code() as i32;
+
+        // Send Shift+Insert (universal paste shortcut)
+        if let Err(e) = remote_desktop_portal::notify_keyboard_keycode(
+            &portal,
+            session,
+            HashMap::new(),
+            shift_keycode,
+            PRESSED_DOWN_STATE,
+        ) {
+            log::error!("input_text_via_clipboard: failed to press Shift: {:?}", e);
+            return;
+        }
+
+        // Press Insert
+        if let Err(e) = remote_desktop_portal::notify_keyboard_keycode(
+            &portal,
+            session,
+            HashMap::new(),
+            insert_keycode,
+            PRESSED_DOWN_STATE,
+        ) {
+            log::error!("input_text_via_clipboard: failed to press Insert: {:?}", e);
+            // Still try to release Shift.
+            // Note: clipboard has already been set by set_clipboard_for_paste_sync but paste
+            // never happened. We don't attempt to restore the previous clipboard contents
+            // because reading the clipboard on Wayland requires focus/permission.
+            let _ = remote_desktop_portal::notify_keyboard_keycode(
+                &portal,
+                session,
+                HashMap::new(),
+                shift_keycode,
+                PRESSED_UP_STATE,
+            );
+            return;
+        }
+
+        // Release Insert
+        if let Err(e) = remote_desktop_portal::notify_keyboard_keycode(
+            &portal,
+            session,
+            HashMap::new(),
+            insert_keycode,
+            PRESSED_UP_STATE,
+        ) {
+            log::error!(
+                "input_text_via_clipboard: failed to release Insert: {:?}",
+                e
+            );
+        }
+
+        // Release Shift
+        if let Err(e) = remote_desktop_portal::notify_keyboard_keycode(
+            &portal,
+            session,
+            HashMap::new(),
+            shift_keycode,
+            PRESSED_UP_STATE,
+        ) {
+            log::error!("input_text_via_clipboard: failed to release Shift: {:?}", e);
         }
     }
 
@@ -196,6 +420,39 @@ pub mod client {
         }
     }
 
+    /// Send a keysym via RemoteDesktop portal.
+    fn send_keysym(
+        keysym: i32,
+        down: bool,
+        conn: Arc<SyncConnection>,
+        session: &Path<'static>,
+    ) -> ResultType<()> {
+        let state: u32 = if down {
+            PRESSED_DOWN_STATE
+        } else {
+            PRESSED_UP_STATE
+        };
+        let portal = get_portal(&conn);
+        log::trace!(
+            "send_keysym: calling notify_keyboard_keysym, keysym={:#x}, state={}",
+            keysym,
+            state
+        );
+        match remote_desktop_portal::notify_keyboard_keysym(
+            &portal,
+            session,
+            HashMap::new(),
+            keysym,
+            state,
+        ) {
+            Ok(_) => {
+                log::trace!("send_keysym: notify_keyboard_keysym succeeded");
+                Ok(())
+            }
+            Err(e) => Err(e.into()),
+        }
+    }
+
     fn get_raw_evdev_keycode(key: u16) -> i32 {
         // 8 is the offset between xkb and evdev
         let mut key = key as i32 - 8;
@@ -231,22 +488,86 @@ pub mod client {
             }
             _ => {
                 if let Ok((key, is_shift)) = map_key(&key) {
-                    if is_shift {
-                        remote_desktop_portal::notify_keyboard_keycode(
+                    let shift_keycode = evdev::Key::KEY_LEFTSHIFT.code() as i32;
+                    if down {
+                        // Press: Shift down first, then key down
+                        if is_shift {
+                            if let Err(e) = remote_desktop_portal::notify_keyboard_keycode(
+                                &portal,
+                                &session,
+                                HashMap::new(),
+                                shift_keycode,
+                                state,
+                            ) {
+                                log::error!("handle_key: failed to press Shift: {:?}", e);
+                                return Err(e.into());
+                            }
+                        }
+                        if let Err(e) = remote_desktop_portal::notify_keyboard_keycode(
                             &portal,
                             &session,
                             HashMap::new(),
-                            evdev::Key::KEY_LEFTSHIFT.code() as i32,
+                            key.code() as i32,
                             state,
-                        )?;
+                        ) {
+                            log::error!("handle_key: failed to press key: {:?}", e);
+                            // Best-effort: release Shift if it was pressed
+                            if is_shift {
+                                if let Err(e) = remote_desktop_portal::notify_keyboard_keycode(
+                                    &portal,
+                                    &session,
+                                    HashMap::new(),
+                                    shift_keycode,
+                                    PRESSED_UP_STATE,
+                                ) {
+                                    log::warn!(
+                                        "handle_key: best-effort Shift release also failed: {:?}",
+                                        e
+                                    );
+                                }
+                            }
+                            return Err(e.into());
+                        }
+                    } else {
+                        // Release: key up first, then Shift up
+                        if let Err(e) = remote_desktop_portal::notify_keyboard_keycode(
+                            &portal,
+                            &session,
+                            HashMap::new(),
+                            key.code() as i32,
+                            PRESSED_UP_STATE,
+                        ) {
+                            log::error!("handle_key: failed to release key: {:?}", e);
+                            // Best-effort: still try to release Shift
+                            if is_shift {
+                                if let Err(e) = remote_desktop_portal::notify_keyboard_keycode(
+                                    &portal,
+                                    &session,
+                                    HashMap::new(),
+                                    shift_keycode,
+                                    PRESSED_UP_STATE,
+                                ) {
+                                    log::warn!(
+                                        "handle_key: best-effort Shift release also failed: {:?}",
+                                        e
+                                    );
+                                }
+                            }
+                            return Err(e.into());
+                        }
+                        if is_shift {
+                            if let Err(e) = remote_desktop_portal::notify_keyboard_keycode(
+                                &portal,
+                                &session,
+                                HashMap::new(),
+                                shift_keycode,
+                                PRESSED_UP_STATE,
+                            ) {
+                                log::error!("handle_key: failed to release Shift: {:?}", e);
+                                return Err(e.into());
+                            }
+                        }
                     }
-                    remote_desktop_portal::notify_keyboard_keycode(
-                        &portal,
-                        &session,
-                        HashMap::new(),
-                        key.code() as i32,
-                        state,
-                    )?;
                 }
             }
         }