move refcounting of key presses to input-emulation (#169)

This commit is contained in:
Ferdinand Schober
2024-08-06 16:46:32 +02:00
committed by GitHub
parent 68361b25d1
commit 8f7890c9be
13 changed files with 225 additions and 218 deletions

View File

@@ -3,19 +3,19 @@ use input_event::Event;
use crate::error::EmulationError;
use super::{EmulationHandle, InputEmulation};
use super::{Emulation, EmulationHandle};
#[derive(Default)]
pub struct DummyEmulation;
pub(crate) struct DummyEmulation;
impl DummyEmulation {
pub fn new() -> Self {
pub(crate) fn new() -> Self {
Self {}
}
}
#[async_trait]
impl InputEmulation for DummyEmulation {
impl Emulation for DummyEmulation {
async fn consume(
&mut self,
event: Event,

View File

@@ -1,31 +1,34 @@
use async_trait::async_trait;
use std::fmt::Display;
use std::{
collections::{HashMap, HashSet},
fmt::Display,
};
use input_event::Event;
use input_event::{Event, KeyboardEvent};
pub use self::error::{EmulationCreationError, EmulationError, InputEmulationError};
#[cfg(windows)]
pub mod windows;
mod windows;
#[cfg(all(unix, feature = "x11", not(target_os = "macos")))]
pub mod x11;
mod x11;
#[cfg(all(unix, feature = "wayland", not(target_os = "macos")))]
pub mod wlroots;
mod wlroots;
#[cfg(all(unix, feature = "xdg_desktop_portal", not(target_os = "macos")))]
pub mod xdg_desktop_portal;
mod xdg_desktop_portal;
#[cfg(all(unix, feature = "libei", not(target_os = "macos")))]
pub mod libei;
mod libei;
#[cfg(target_os = "macos")]
pub mod macos;
mod macos;
/// fallback input emulation (logs events)
pub mod dummy;
pub mod error;
mod dummy;
mod error;
pub type EmulationHandle = u64;
@@ -66,8 +69,164 @@ impl Display for Backend {
}
}
pub struct InputEmulation {
emulation: Box<dyn Emulation>,
handles: HashSet<EmulationHandle>,
pressed_keys: HashMap<EmulationHandle, HashSet<u32>>,
}
impl InputEmulation {
async fn with_backend(backend: Backend) -> Result<InputEmulation, EmulationCreationError> {
let emulation: Box<dyn Emulation> = match backend {
#[cfg(all(unix, feature = "wayland", not(target_os = "macos")))]
Backend::Wlroots => Box::new(wlroots::WlrootsEmulation::new()?),
#[cfg(all(unix, feature = "libei", not(target_os = "macos")))]
Backend::Libei => Box::new(libei::LibeiEmulation::new().await?),
#[cfg(all(unix, feature = "x11", not(target_os = "macos")))]
Backend::X11 => Box::new(x11::X11Emulation::new()?),
#[cfg(all(unix, feature = "xdg_desktop_portal", not(target_os = "macos")))]
Backend::Xdp => Box::new(xdg_desktop_portal::DesktopPortalEmulation::new().await?),
#[cfg(windows)]
Backend::Windows => Box::new(windows::WindowsEmulation::new()?),
#[cfg(target_os = "macos")]
Backend::MacOs => Box::new(macos::MacOSEmulation::new()?),
Backend::Dummy => Box::new(dummy::DummyEmulation::new()),
};
Ok(Self {
emulation,
handles: HashSet::new(),
pressed_keys: HashMap::new(),
})
}
pub async fn new(backend: Option<Backend>) -> Result<InputEmulation, EmulationCreationError> {
if let Some(backend) = backend {
let b = Self::with_backend(backend).await;
if b.is_ok() {
log::info!("using emulation backend: {backend}");
}
return b;
}
for backend in [
#[cfg(all(unix, feature = "wayland", not(target_os = "macos")))]
Backend::Wlroots,
#[cfg(all(unix, feature = "libei", not(target_os = "macos")))]
Backend::Libei,
#[cfg(all(unix, feature = "xdg_desktop_portal", not(target_os = "macos")))]
Backend::Xdp,
#[cfg(all(unix, feature = "x11", not(target_os = "macos")))]
Backend::X11,
#[cfg(windows)]
Backend::Windows,
#[cfg(target_os = "macos")]
Backend::MacOs,
Backend::Dummy,
] {
match Self::with_backend(backend).await {
Ok(b) => {
log::info!("using emulation backend: {backend}");
return Ok(b);
}
Err(e) if e.cancelled_by_user() => return Err(e),
Err(e) => log::warn!("{e}"),
}
}
Err(EmulationCreationError::NoAvailableBackend)
}
pub async fn consume(
&mut self,
event: Event,
handle: EmulationHandle,
) -> Result<(), EmulationError> {
match event {
Event::Keyboard(KeyboardEvent::Key { key, state, .. }) => {
// prevent double pressed / released keys
if self.update_pressed_keys(handle, key, state) {
self.emulation.consume(event, handle).await?;
}
Ok(())
}
_ => self.emulation.consume(event, handle).await,
}
}
pub async fn create(&mut self, handle: EmulationHandle) -> bool {
if self.handles.insert(handle) {
self.emulation.create(handle).await;
true
} else {
false
}
}
pub async fn destroy(&mut self, handle: EmulationHandle) {
let _ = self.release_keys(handle).await;
if self.handles.remove(&handle) {
self.emulation.destroy(handle).await
}
}
pub async fn terminate(&mut self) {
for handle in self.handles.iter().cloned().collect::<Vec<_>>() {
self.destroy(handle).await
}
self.emulation.terminate().await
}
pub async fn release_keys(&mut self, handle: EmulationHandle) -> Result<(), EmulationError> {
if let Some(keys) = self.pressed_keys.get_mut(&handle) {
let keys = keys.drain().collect::<Vec<_>>();
for key in keys {
let event = Event::Keyboard(KeyboardEvent::Key {
time: 0,
key,
state: 0,
});
self.emulation.consume(event, handle).await?;
if let Ok(key) = input_event::scancode::Linux::try_from(key) {
log::warn!("releasing stuck key: {key:?}");
}
}
}
let event = Event::Keyboard(KeyboardEvent::Modifiers {
mods_depressed: 0,
mods_latched: 0,
mods_locked: 0,
group: 0,
});
self.emulation.consume(event, handle).await?;
Ok(())
}
pub fn has_pressed_keys(&self, handle: EmulationHandle) -> bool {
self.pressed_keys
.get(&handle)
.is_some_and(|p| !p.is_empty())
}
/// update the pressed_keys for the given handle
/// returns whether the event should be processed
fn update_pressed_keys(&mut self, handle: EmulationHandle, key: u32, state: u8) -> bool {
let Some(pressed_keys) = self.pressed_keys.get_mut(&handle) else {
return false;
};
if state == 0 {
// currently pressed => can release
pressed_keys.remove(&key)
} else {
// currently not pressed => can press
pressed_keys.insert(key)
}
}
}
#[async_trait]
pub trait InputEmulation: Send {
trait Emulation: Send {
async fn consume(
&mut self,
event: Event,
@@ -77,64 +236,3 @@ pub trait InputEmulation: Send {
async fn destroy(&mut self, handle: EmulationHandle);
async fn terminate(&mut self);
}
pub async fn create_backend(
backend: Backend,
) -> Result<Box<dyn InputEmulation>, EmulationCreationError> {
match backend {
#[cfg(all(unix, feature = "wayland", not(target_os = "macos")))]
Backend::Wlroots => Ok(Box::new(wlroots::WlrootsEmulation::new()?)),
#[cfg(all(unix, feature = "libei", not(target_os = "macos")))]
Backend::Libei => Ok(Box::new(libei::LibeiEmulation::new().await?)),
#[cfg(all(unix, feature = "x11", not(target_os = "macos")))]
Backend::X11 => Ok(Box::new(x11::X11Emulation::new()?)),
#[cfg(all(unix, feature = "xdg_desktop_portal", not(target_os = "macos")))]
Backend::Xdp => Ok(Box::new(
xdg_desktop_portal::DesktopPortalEmulation::new().await?,
)),
#[cfg(windows)]
Backend::Windows => Ok(Box::new(windows::WindowsEmulation::new()?)),
#[cfg(target_os = "macos")]
Backend::MacOs => Ok(Box::new(macos::MacOSEmulation::new()?)),
Backend::Dummy => Ok(Box::new(dummy::DummyEmulation::new())),
}
}
pub async fn create(
backend: Option<Backend>,
) -> Result<Box<dyn InputEmulation>, EmulationCreationError> {
if let Some(backend) = backend {
let b = create_backend(backend).await;
if b.is_ok() {
log::info!("using emulation backend: {backend}");
}
return b;
}
for backend in [
#[cfg(all(unix, feature = "wayland", not(target_os = "macos")))]
Backend::Wlroots,
#[cfg(all(unix, feature = "libei", not(target_os = "macos")))]
Backend::Libei,
#[cfg(all(unix, feature = "xdg_desktop_portal", not(target_os = "macos")))]
Backend::Xdp,
#[cfg(all(unix, feature = "x11", not(target_os = "macos")))]
Backend::X11,
#[cfg(windows)]
Backend::Windows,
#[cfg(target_os = "macos")]
Backend::MacOs,
Backend::Dummy,
] {
match create_backend(backend).await {
Ok(b) => {
log::info!("using emulation backend: {backend}");
return Ok(b);
}
Err(e) if e.cancelled_by_user() => return Err(e),
Err(e) => log::warn!("{e}"),
}
}
Err(EmulationCreationError::NoAvailableBackend)
}

View File

@@ -34,7 +34,7 @@ use input_event::{Event, KeyboardEvent, PointerEvent};
use crate::error::{EmulationError, ReisConvertStreamError};
use super::{error::LibeiEmulationCreationError, EmulationHandle, InputEmulation};
use super::{error::LibeiEmulationCreationError, Emulation, EmulationHandle};
static INTERFACES: Lazy<HashMap<&'static str, u32>> = Lazy::new(|| {
let mut m = HashMap::new();
@@ -60,7 +60,7 @@ struct Devices {
keyboard: Arc<RwLock<Option<(ei::Device, ei::Keyboard)>>>,
}
pub struct LibeiEmulation<'a> {
pub(crate) struct LibeiEmulation<'a> {
context: ei::Context,
devices: Devices,
ei_task: JoinHandle<()>,
@@ -99,7 +99,7 @@ async fn get_ei_fd<'a>(
}
impl<'a> LibeiEmulation<'a> {
pub async fn new() -> Result<Self, LibeiEmulationCreationError> {
pub(crate) async fn new() -> Result<Self, LibeiEmulationCreationError> {
let (_remote_desktop, session, eifd) = get_ei_fd().await?;
let stream = UnixStream::from(eifd);
stream.set_nonblocking(true)?;
@@ -147,7 +147,7 @@ impl<'a> Drop for LibeiEmulation<'a> {
}
#[async_trait]
impl<'a> InputEmulation for LibeiEmulation<'a> {
impl<'a> Emulation for LibeiEmulation<'a> {
async fn consume(
&mut self,
event: Event,

View File

@@ -1,4 +1,4 @@
use super::{error::EmulationError, EmulationHandle, InputEmulation};
use super::{error::EmulationError, Emulation, EmulationHandle};
use async_trait::async_trait;
use core_graphics::display::{CGDisplayBounds, CGMainDisplayID, CGPoint};
use core_graphics::event::{
@@ -16,8 +16,8 @@ use super::error::MacOSEmulationCreationError;
const DEFAULT_REPEAT_DELAY: Duration = Duration::from_millis(500);
const DEFAULT_REPEAT_INTERVAL: Duration = Duration::from_millis(32);
pub struct MacOSEmulation {
pub event_source: CGEventSource,
pub(crate) struct MacOSEmulation {
event_source: CGEventSource,
repeat_task: Option<AbortHandle>,
button_state: ButtonState,
}
@@ -53,7 +53,7 @@ impl IndexMut<CGMouseButton> for ButtonState {
unsafe impl Send for MacOSEmulation {}
impl MacOSEmulation {
pub fn new() -> Result<Self, MacOSEmulationCreationError> {
pub(crate) fn new() -> Result<Self, MacOSEmulationCreationError> {
let event_source = CGEventSource::new(CGEventSourceStateID::CombinedSessionState)
.map_err(|_| MacOSEmulationCreationError::EventSourceCreation)?;
let button_state = ButtonState {
@@ -106,7 +106,7 @@ fn key_event(event_source: CGEventSource, key: u16, state: u8) {
}
#[async_trait]
impl InputEmulation for MacOSEmulation {
impl Emulation for MacOSEmulation {
async fn consume(
&mut self,
event: Event,

View File

@@ -19,23 +19,23 @@ use windows::Win32::UI::Input::KeyboardAndMouse::{
};
use windows::Win32::UI::WindowsAndMessaging::{XBUTTON1, XBUTTON2};
use super::{EmulationHandle, InputEmulation};
use super::{Emulation, EmulationHandle};
const DEFAULT_REPEAT_DELAY: Duration = Duration::from_millis(500);
const DEFAULT_REPEAT_INTERVAL: Duration = Duration::from_millis(32);
pub struct WindowsEmulation {
pub(crate) struct WindowsEmulation {
repeat_task: Option<AbortHandle>,
}
impl WindowsEmulation {
pub fn new() -> Result<Self, WindowsEmulationCreationError> {
pub(crate) fn new() -> Result<Self, WindowsEmulationCreationError> {
Ok(Self { repeat_task: None })
}
}
#[async_trait]
impl InputEmulation for WindowsEmulation {
impl Emulation for WindowsEmulation {
async fn consume(&mut self, event: Event, _: EmulationHandle) -> Result<(), EmulationError> {
match event {
Event::Pointer(pointer_event) => match pointer_event {

View File

@@ -1,6 +1,6 @@
use crate::error::EmulationError;
use super::{error::WlrootsEmulationCreationError, InputEmulation};
use super::{error::WlrootsEmulationCreationError, Emulation};
use async_trait::async_trait;
use std::collections::HashMap;
use std::io;
@@ -50,7 +50,7 @@ pub(crate) struct WlrootsEmulation {
}
impl WlrootsEmulation {
pub fn new() -> Result<Self, WlrootsEmulationCreationError> {
pub(crate) fn new() -> Result<Self, WlrootsEmulationCreationError> {
let conn = Connection::connect_to_env()?;
let (globals, queue) = registry_queue_init::<State>(&conn)?;
let qh = queue.handle();
@@ -116,7 +116,7 @@ impl State {
}
#[async_trait]
impl InputEmulation for WlrootsEmulation {
impl Emulation for WlrootsEmulation {
async fn consume(
&mut self,
event: Event,

View File

@@ -11,16 +11,16 @@ use input_event::{
use crate::error::EmulationError;
use super::{error::X11EmulationCreationError, EmulationHandle, InputEmulation};
use super::{error::X11EmulationCreationError, Emulation, EmulationHandle};
pub struct X11Emulation {
pub(crate) struct X11Emulation {
display: *mut xlib::Display,
}
unsafe impl Send for X11Emulation {}
impl X11Emulation {
pub fn new() -> Result<Self, X11EmulationCreationError> {
pub(crate) fn new() -> Result<Self, X11EmulationCreationError> {
let display = unsafe {
match xlib::XOpenDisplay(ptr::null()) {
d if d == ptr::null::<xlib::Display>() as *mut xlib::Display => {
@@ -99,7 +99,7 @@ impl Drop for X11Emulation {
}
#[async_trait]
impl InputEmulation for X11Emulation {
impl Emulation for X11Emulation {
async fn consume(&mut self, event: Event, _: EmulationHandle) -> Result<(), EmulationError> {
match event {
Event::Pointer(pointer_event) => match pointer_event {

View File

@@ -16,15 +16,15 @@ use input_event::{
use crate::error::EmulationError;
use super::{error::XdpEmulationCreationError, EmulationHandle, InputEmulation};
use super::{error::XdpEmulationCreationError, Emulation, EmulationHandle};
pub struct DesktopPortalEmulation<'a> {
pub(crate) struct DesktopPortalEmulation<'a> {
proxy: RemoteDesktop<'a>,
session: Session<'a, RemoteDesktop<'a>>,
}
impl<'a> DesktopPortalEmulation<'a> {
pub async fn new() -> Result<DesktopPortalEmulation<'a>, XdpEmulationCreationError> {
pub(crate) async fn new() -> Result<DesktopPortalEmulation<'a>, XdpEmulationCreationError> {
log::debug!("connecting to org.freedesktop.portal.RemoteDesktop portal ...");
let proxy = RemoteDesktop::new().await?;
@@ -56,7 +56,7 @@ impl<'a> DesktopPortalEmulation<'a> {
}
#[async_trait]
impl<'a> InputEmulation for DesktopPortalEmulation<'a> {
impl<'a> Emulation for DesktopPortalEmulation<'a> {
async fn consume(
&mut self,
event: input_event::Event,

View File

@@ -125,8 +125,8 @@ pub struct ClientState {
/// e.g. Laptops usually have at least an ethernet and a wifi port
/// which have different ip addresses
pub ips: HashSet<IpAddr>,
/// keys currently pressed by this client
pub pressed_keys: HashSet<u32>,
/// client has pressed keys
pub has_pressed_keys: bool,
/// dns resolving in progress
pub resolving: bool,
}

View File

@@ -1,5 +1,6 @@
use crate::config::Config;
use anyhow::Result;
use input_emulation::InputEmulation;
use input_event::{Event, PointerEvent};
use std::f64::consts::PI;
use std::time::{Duration, Instant};
@@ -22,7 +23,7 @@ const RADIUS: f64 = 100.0;
async fn input_emulation_test(config: Config) -> Result<()> {
let backend = config.emulation_backend.map(|b| b.into());
let mut emulation = input_emulation::create(backend).await?;
let mut emulation = InputEmulation::new(backend).await?;
emulation.create(0).await;
let start = Instant::now();
let mut offset = (0, 0);

View File

@@ -441,6 +441,12 @@ impl Server {
}
}
fn update_pressed_keys(&self, handle: ClientHandle, has_pressed_keys: bool) {
if let Some((_, s)) = self.client_manager.borrow_mut().get_mut(handle) {
s.has_pressed_keys = has_pressed_keys;
}
}
fn update_fix_ips(&self, handle: ClientHandle, fix_ips: Vec<IpAddr>) {
if let Some((c, _)) = self.client_manager.borrow_mut().get_mut(handle) {
c.fix_ips = fix_ips;

View File

@@ -11,7 +11,7 @@ use crate::{
server::State,
};
use input_emulation::{self, EmulationError, EmulationHandle, InputEmulation, InputEmulationError};
use input_event::{Event, KeyboardEvent};
use input_event::Event;
use super::{network_task::NetworkError, CaptureEvent, Server};
@@ -73,9 +73,7 @@ async fn do_emulation(
let backend = server.config.emulation_backend.map(|b| b.into());
log::info!("creating input emulation...");
let mut emulation = tokio::select! {
r = input_emulation::create(backend) => {
r?
}
r = InputEmulation::new(backend) => r?,
_ = server.cancelled() => return Ok(()),
};
@@ -87,19 +85,13 @@ async fn do_emulation(
}
let res = do_emulation_session(server, &mut emulation, rx, udp_rx, sender_tx, capture_tx).await;
// release potentially still pressed keys
release_all_keys(server, &mut emulation).await?;
emulation.terminate().await;
res?;
Ok(())
emulation.terminate().await; // manual drop
res
}
async fn do_emulation_session(
server: &Server,
emulation: &mut Box<dyn InputEmulation>,
emulation: &mut InputEmulation,
rx: &mut Receiver<EmulationEvent>,
udp_rx: &mut Receiver<Result<(Event, SocketAddr), NetworkError>>,
sender_tx: &Sender<(Event, SocketAddr)>,
@@ -121,9 +113,9 @@ async fn do_emulation_session(
}
emulate_event = rx.recv() => {
match emulate_event.expect("channel closed") {
EmulationEvent::Create(h) => emulation.create(h).await,
EmulationEvent::Create(h) => { let _ = emulation.create(h).await; },
EmulationEvent::Destroy(h) => emulation.destroy(h).await,
EmulationEvent::ReleaseKeys(c) => release_keys(server, emulation, c).await?,
EmulationEvent::ReleaseKeys(c) => emulation.release_keys(c).await?,
}
}
_ = server.notifies.cancel.cancelled() => break Ok(()),
@@ -134,7 +126,7 @@ async fn do_emulation_session(
async fn handle_udp_rx(
server: &Server,
capture_tx: &Sender<CaptureEvent>,
emulate: &mut Box<dyn InputEmulation>,
emulate: &mut InputEmulation,
sender_tx: &Sender<(Event, SocketAddr)>,
last_ignored: &mut Option<SocketAddr>,
event: (Event, SocketAddr),
@@ -155,9 +147,7 @@ async fn handle_udp_rx(
(Event::Ping(), addr) => {
let _ = sender_tx.send((Event::Pong(), addr)).await;
}
(Event::Disconnect(), _) => {
release_keys(server, emulate, handle).await?;
}
(Event::Disconnect(), _) => emulate.release_keys(handle).await?,
(event, addr) => {
// tell clients that we are ready to receive events
if let Event::Enter() = event {
@@ -177,27 +167,12 @@ async fn handle_udp_rx(
}
}
State::Receiving => {
let ignore_event =
if let Event::Keyboard(KeyboardEvent::Key { key, state, .. }) = event {
let (ignore_event, restart_timer) = update_client_keys(
&mut server.client_manager.borrow_mut(),
handle,
key,
state,
);
// restart timer if necessary
if restart_timer {
server.restart_ping_timer();
}
ignore_event
} else {
false
};
// workaround buggy rdp backend.
if !ignore_event {
// consume event
emulate.consume(event, handle).await?;
log::trace!("{event} => emulate");
log::trace!("{event} => emulate");
emulate.consume(event, handle).await?;
let has_pressed_keys = emulate.has_pressed_keys(handle);
server.update_pressed_keys(handle, has_pressed_keys);
if has_pressed_keys {
server.restart_ping_timer();
}
}
State::AwaitingLeave => {
@@ -224,57 +199,6 @@ async fn handle_udp_rx(
Ok(())
}
async fn release_all_keys(
server: &Server,
emulation: &mut Box<dyn InputEmulation>,
) -> Result<(), EmulationError> {
let clients = server
.client_manager
.borrow()
.get_client_states()
.map(|(h, _)| h)
.collect::<Vec<_>>();
for client in clients {
release_keys(server, emulation, client).await?;
}
Ok(())
}
async fn release_keys(
server: &Server,
emulate: &mut Box<dyn InputEmulation>,
client: ClientHandle,
) -> Result<(), EmulationError> {
let keys = server
.client_manager
.borrow_mut()
.get_mut(client)
.iter_mut()
.flat_map(|(_, s)| s.pressed_keys.drain())
.collect::<Vec<_>>();
for key in keys {
let event = Event::Keyboard(KeyboardEvent::Key {
time: 0,
key,
state: 0,
});
emulate.consume(event, client).await?;
if let Ok(key) = input_event::scancode::Linux::try_from(key) {
log::warn!("releasing stuck key: {key:?}");
}
}
let event = Event::Keyboard(KeyboardEvent::Modifiers {
mods_depressed: 0,
mods_latched: 0,
mods_locked: 0,
group: 0,
});
emulate.consume(event, client).await?;
Ok(())
}
fn activate_client_if_exists(
client_manager: &mut ClientManager,
addr: SocketAddr,
@@ -299,25 +223,3 @@ fn activate_client_if_exists(
client_state.active_addr = Some(addr);
Some(handle)
}
fn update_client_keys(
client_manager: &mut ClientManager,
handle: ClientHandle,
key: u32,
state: u8,
) -> (bool, bool) {
let Some(client_state) = client_manager.get_mut(handle).map(|(_, s)| s) else {
return (true, false);
};
// ignore double press / release events
let ignore_event = if state == 0 {
// ignore release event if key not pressed
!client_state.pressed_keys.remove(&key)
} else {
// ignore press event if key not released
!client_state.pressed_keys.insert(key)
};
let restart_timer = !client_state.pressed_keys.is_empty();
(ignore_event, restart_timer)
}

View File

@@ -42,8 +42,8 @@ async fn ping_task(
let ping_clients: Vec<ClientHandle> = if receiving {
// if receiving we care about clients with pressed keys
client_manager
.get_client_states_mut()
.filter(|(_, (_, s))| !s.pressed_keys.is_empty())
.get_client_states()
.filter(|(_, (_, s))| s.has_pressed_keys)
.map(|(h, _)| h)
.collect()
} else {