Allow input capture & emulation being disabled (#158)

* Input capture and emulation can now be disabled and will prompt the user to enable again.

* Improved error handling to deliver more useful error messages
This commit is contained in:
Ferdinand Schober
2024-07-16 20:34:46 +02:00
committed by GitHub
parent 55bdf1e63e
commit bea7d6f8a5
41 changed files with 2079 additions and 1543 deletions

View File

@@ -1,14 +1,14 @@
use ashpd::{
desktop::{
input_capture::{Activated, Barrier, BarrierID, Capabilities, InputCapture, Region, Zones},
ResponseError, Session,
Session,
},
enumflags2::BitFlags,
};
use futures::StreamExt;
use async_trait::async_trait;
use futures::{FutureExt, StreamExt};
use reis::{
ei::{self, keyboard::KeyState},
eis::button::ButtonState,
ei,
event::{DeviceCapability, EiEvent},
tokio::{EiConvertEventStream, EiEventStream},
};
@@ -19,27 +19,35 @@ use std::{
os::unix::net::UnixStream,
pin::Pin,
rc::Rc,
task::{ready, Context, Poll},
sync::Arc,
task::{Context, Poll},
};
use tokio::{
sync::mpsc::{Receiver, Sender},
sync::{
mpsc::{self, Receiver, Sender},
Notify,
},
task::JoinHandle,
};
use tokio_util::sync::CancellationToken;
use futures_core::Stream;
use once_cell::sync::Lazy;
use input_event::{Event, KeyboardEvent, PointerEvent};
use crate::error::{CaptureError, ReisConvertEventStreamError};
use input_event::Event;
use super::{
error::LibeiCaptureCreationError, CaptureHandle, InputCapture as LanMouseInputCapture, Position,
error::{CaptureError, LibeiCaptureCreationError, ReisConvertEventStreamError},
CaptureHandle, InputCapture as LanMouseInputCapture, Position,
};
#[derive(Debug)]
enum ProducerEvent {
Release,
/* there is a bug in xdg-remote-desktop-portal-gnome / mutter that
* prevents receiving further events after a session has been disabled once.
* Therefore the session needs to be recreated when the barriers are updated */
/// events that necessitate restarting the capture session
#[derive(Clone, Copy, Debug)]
enum LibeiNotifyEvent {
Create(CaptureHandle, Position),
Destroy(CaptureHandle),
}
@@ -47,9 +55,12 @@ enum ProducerEvent {
#[allow(dead_code)]
pub struct LibeiInputCapture<'a> {
input_capture: Pin<Box<InputCapture<'a>>>,
libei_task: JoinHandle<Result<(), CaptureError>>,
event_rx: tokio::sync::mpsc::Receiver<(CaptureHandle, Event)>,
notify_tx: tokio::sync::mpsc::Sender<ProducerEvent>,
capture_task: JoinHandle<Result<(), CaptureError>>,
event_rx: Receiver<(CaptureHandle, Event)>,
notify_capture: Sender<LibeiNotifyEvent>,
notify_release: Arc<Notify>,
cancellation_token: CancellationToken,
terminated: bool,
}
static INTERFACES: Lazy<HashMap<&'static str, u32>> = Lazy::new(|| {
@@ -68,14 +79,15 @@ static INTERFACES: Lazy<HashMap<&'static str, u32>> = Lazy::new(|| {
m
});
/// returns (start pos, end pos), inclusive
fn pos_to_barrier(r: &Region, pos: Position) -> (i32, i32, i32, i32) {
let (x, y) = (r.x_offset(), r.y_offset());
let (width, height) = (r.width() as i32, r.height() as i32);
let (w, h) = (r.width() as i32, r.height() as i32);
match pos {
Position::Left => (x, y, x, y + height - 1), // start pos, end pos, inclusive
Position::Right => (x + width, y, x + width, y + height - 1),
Position::Top => (x, y, x + width - 1, y),
Position::Bottom => (x, y + height, x + width - 1, y + height),
Position::Left => (x, y, x, y + h - 1),
Position::Right => (x + w, y, x + w, y + h - 1),
Position::Top => (x, y, x + w - 1, y),
Position::Bottom => (x, y + h, x + w - 1, y + h),
}
}
@@ -125,31 +137,16 @@ async fn update_barriers(
Ok(id_map)
}
impl<'a> Drop for LibeiInputCapture<'a> {
fn drop(&mut self) {
self.libei_task.abort();
}
}
async fn create_session<'a>(
input_capture: &'a InputCapture<'a>,
) -> std::result::Result<(Session<'a>, BitFlags<Capabilities>), ashpd::Error> {
log::debug!("creating input capture session");
let (session, capabilities) = loop {
match input_capture
.create_session(
&ashpd::WindowIdentifier::default(),
Capabilities::Keyboard | Capabilities::Pointer | Capabilities::Touchscreen,
)
.await
{
Ok(s) => break s,
Err(ashpd::Error::Response(ResponseError::Cancelled)) => continue,
o => o?,
};
};
log::debug!("capabilities: {capabilities:?}");
Ok((session, capabilities))
input_capture
.create_session(
&ashpd::WindowIdentifier::default(),
Capabilities::Keyboard | Capabilities::Pointer | Capabilities::Touchscreen,
)
.await
}
async fn connect_to_eis(
@@ -182,6 +179,7 @@ async fn libei_event_handler(
mut ei_event_stream: EiConvertEventStream,
context: ei::Context,
event_tx: Sender<(CaptureHandle, Event)>,
release_session: Arc<Notify>,
current_client: Rc<Cell<Option<CaptureHandle>>>,
) -> Result<(), CaptureError> {
loop {
@@ -192,20 +190,7 @@ async fn libei_event_handler(
.map_err(ReisConvertEventStreamError::from)?;
log::trace!("from ei: {ei_event:?}");
let client = current_client.get();
handle_ei_event(ei_event, client, &context, &event_tx).await;
}
}
async fn wait_for_active_client(
notify_rx: &mut Receiver<ProducerEvent>,
active_clients: &mut Vec<(CaptureHandle, Position)>,
) {
// wait for a client update
while let Some(producer_event) = notify_rx.recv().await {
if let ProducerEvent::Create(c, p) = producer_event {
handle_producer_event(ProducerEvent::Create(c, p), active_clients);
break;
}
handle_ei_event(ei_event, client, &context, &event_tx, &release_session).await?;
}
}
@@ -215,16 +200,30 @@ impl<'a> LibeiInputCapture<'a> {
let input_capture_ptr = input_capture.as_ref().get_ref() as *const InputCapture<'static>;
let first_session = Some(create_session(unsafe { &*input_capture_ptr }).await?);
let (event_tx, event_rx) = tokio::sync::mpsc::channel(32);
let (notify_tx, notify_rx) = tokio::sync::mpsc::channel(32);
let capture = do_capture(input_capture_ptr, notify_rx, first_session, event_tx);
let libei_task = tokio::task::spawn_local(capture);
let (event_tx, event_rx) = mpsc::channel(1);
let (notify_capture, notify_rx) = mpsc::channel(1);
let notify_release = Arc::new(Notify::new());
let cancellation_token = CancellationToken::new();
let capture = do_capture(
input_capture_ptr,
notify_rx,
notify_release.clone(),
first_session,
event_tx,
cancellation_token.clone(),
);
let capture_task = tokio::task::spawn_local(capture);
let producer = Self {
input_capture,
event_rx,
libei_task,
notify_tx,
capture_task,
notify_capture,
notify_release,
cancellation_token,
terminated: false,
};
Ok(producer)
@@ -232,67 +231,157 @@ impl<'a> LibeiInputCapture<'a> {
}
async fn do_capture<'a>(
input_capture_ptr: *const InputCapture<'static>,
mut notify_rx: Receiver<ProducerEvent>,
mut first_session: Option<(Session<'a>, BitFlags<Capabilities>)>,
input_capture: *const InputCapture<'a>,
mut capture_event: Receiver<LibeiNotifyEvent>,
notify_release: Arc<Notify>,
session: Option<(Session<'a>, BitFlags<Capabilities>)>,
event_tx: Sender<(CaptureHandle, Event)>,
cancellation_token: CancellationToken,
) -> Result<(), CaptureError> {
/* safety: libei_task does not outlive Self */
let input_capture = unsafe { &*input_capture_ptr };
let mut session = session.map(|s| s.0);
/* safety: libei_task does not outlive Self */
let input_capture = unsafe { &*input_capture };
let mut active_clients: Vec<(CaptureHandle, Position)> = vec![];
let mut next_barrier_id = 1u32;
/* there is a bug in xdg-remote-desktop-portal-gnome / mutter that
* prevents receiving further events after a session has been disabled once.
* Therefore the session needs to recreated when the barriers are updated */
let mut zones_changed = input_capture.receive_zones_changed().await?;
loop {
// otherwise it asks to capture input even with no active clients
if active_clients.is_empty() {
wait_for_active_client(&mut notify_rx, &mut active_clients).await;
if notify_rx.is_closed() {
break Ok(());
} else {
continue;
// do capture session
let cancel_session = CancellationToken::new();
let cancel_update = CancellationToken::new();
let mut capture_event_occured: Option<LibeiNotifyEvent> = None;
let mut zones_have_changed = false;
// kill session if clients need to be updated
let handle_session_update_request = async {
tokio::select! {
_ = cancellation_token.cancelled() => {
log::debug!("cancelled")
}, /* exit requested */
_ = cancel_update.cancelled() => {
log::debug!("update task cancelled");
}, /* session exited */
_ = zones_changed.next() => {
log::debug!("zones changed!");
zones_have_changed = true
}, /* zones have changed */
e = capture_event.recv() => if let Some(e) = e { /* clients changed */
log::debug!("capture event: {e:?}");
capture_event_occured.replace(e);
},
}
// kill session (might already be dead!)
log::debug!("=> cancelling session");
cancel_session.cancel();
};
if !active_clients.is_empty() {
// create session
let mut session = match session.take() {
Some(s) => s,
None => create_session(input_capture).await?.0,
};
let capture_session = do_capture_session(
input_capture,
&mut session,
&event_tx,
&mut active_clients,
&mut next_barrier_id,
&notify_release,
(cancel_session.clone(), cancel_update.clone()),
);
let (capture_result, ()) = tokio::join!(capture_session, handle_session_update_request);
log::debug!("capture session + session_update task done!");
// disable capture
log::debug!("disabling input capture");
if let Err(e) = input_capture.disable(&session).await {
log::warn!("input_capture.disable(&session) {e}");
}
if let Err(e) = session.close().await {
log::warn!("session.close(): {e}");
}
// propagate error from capture session
capture_result?;
} else {
handle_session_update_request.await;
}
// update clients if requested
if let Some(event) = capture_event_occured.take() {
match event {
LibeiNotifyEvent::Create(c, p) => active_clients.push((c, p)),
LibeiNotifyEvent::Destroy(c) => active_clients.retain(|(h, _)| *h != c),
}
}
let current_client = Rc::new(Cell::new(None));
// break
if cancellation_token.is_cancelled() {
break Ok(());
}
}
}
// create session
let (session, _) = match first_session.take() {
Some(s) => s,
_ => create_session(input_capture).await?,
};
async fn do_capture_session(
input_capture: &InputCapture<'_>,
session: &mut Session<'_>,
event_tx: &Sender<(CaptureHandle, Event)>,
active_clients: &mut Vec<(CaptureHandle, Position)>,
next_barrier_id: &mut u32,
notify_release: &Notify,
cancel: (CancellationToken, CancellationToken),
) -> Result<(), CaptureError> {
let (cancel_session, cancel_update) = cancel;
// current client
let current_client = Rc::new(Cell::new(None));
// connect to eis server
let (context, ei_event_stream) = connect_to_eis(input_capture, &session).await?;
// connect to eis server
let (context, ei_event_stream) = connect_to_eis(input_capture, session).await?;
// async event task
let mut ei_task: JoinHandle<Result<(), CaptureError>> =
tokio::task::spawn_local(libei_event_handler(
// set barriers
let client_for_barrier_id =
update_barriers(input_capture, session, active_clients, next_barrier_id).await?;
log::debug!("enabling session");
input_capture.enable(session).await?;
// cancellation token to release session
let release_session = Arc::new(Notify::new());
// async event task
let cancel_ei_handler = CancellationToken::new();
let event_chan = event_tx.clone();
let client = current_client.clone();
let cancel_session_clone = cancel_session.clone();
let release_session_clone = release_session.clone();
let cancel_ei_handler_clone = cancel_ei_handler.clone();
let ei_task = async move {
tokio::select! {
r = libei_event_handler(
ei_event_stream,
context,
event_tx.clone(),
current_client.clone(),
));
event_chan,
release_session_clone,
client,
) => {
log::debug!("libei exited: {r:?} cancelling session task");
cancel_session_clone.cancel();
}
_ = cancel_ei_handler_clone.cancelled() => {},
}
Ok::<(), CaptureError>(())
};
let capture_session_task = async {
// receiver for activation tokens
let mut activated = input_capture.receive_activated().await?;
let mut zones_changed = input_capture.receive_zones_changed().await?;
// set barriers
let client_for_barrier_id = update_barriers(
input_capture,
&session,
&active_clients,
&mut next_barrier_id,
)
.await?;
log::debug!("enabling session");
input_capture.enable(&session).await?;
let mut ei_devices_changed = false;
loop {
tokio::select! {
activated = activated.next() => {
@@ -304,57 +393,60 @@ async fn do_capture<'a>(
.expect("invalid barrier id");
current_client.replace(Some(client));
if event_tx.send((client, Event::Enter())).await.is_err() {
break;
};
// client entered => send event
event_tx.send((client, Event::Enter())).await.expect("no channel");
tokio::select! {
producer_event = notify_rx.recv() => {
let producer_event = producer_event.expect("channel closed");
if handle_producer_event(producer_event, &mut active_clients) {
break; /* clients updated */
}
}
zones_changed = zones_changed.next() => {
log::debug!("zones changed: {zones_changed:?}");
break;
}
res = &mut ei_task => {
if let Err(e) = res.expect("ei task paniced") {
log::warn!("libei task exited: {e}");
}
break;
}
_ = notify_release.notified() => { /* capture release */
log::debug!("release session requested");
},
_ = release_session.notified() => { /* release session */
log::debug!("ei devices changed");
ei_devices_changed = true;
},
_ = cancel_session.cancelled() => { /* kill session notify */
log::debug!("session cancel requested");
break
},
}
release_capture(
input_capture,
&session,
activated,
client,
&active_clients,
).await?;
release_capture(input_capture, session, activated, client, active_clients).await?;
}
producer_event = notify_rx.recv() => {
let producer_event = producer_event.expect("channel closed");
if handle_producer_event(producer_event, &mut active_clients) {
/* clients updated */
break;
}
_ = notify_release.notified() => { /* capture release -> we are not capturing anyway, so ignore */
log::debug!("release session requested");
},
res = &mut ei_task => {
if let Err(e) = res.expect("ei task paniced") {
log::warn!("libei task exited: {e}");
}
break;
}
_ = release_session.notified() => { /* release session */
log::debug!("ei devices changed");
ei_devices_changed = true;
},
_ = cancel_session.cancelled() => { /* kill session notify */
log::debug!("session cancel requested");
break
},
}
if ei_devices_changed {
/* for whatever reason, GNOME seems to kill the session
* as soon as devices are added or removed, so we need
* to cancel */
break;
}
}
ei_task.abort();
input_capture.disable(&session).await?;
if event_tx.is_closed() {
break Ok(());
}
}
// cancel libei task
log::debug!("session exited: killing libei task");
cancel_ei_handler.cancel();
Ok::<(), CaptureError>(())
};
let (a, b) = tokio::join!(ei_task, capture_session_task);
cancel_update.cancel();
log::debug!("both session and ei task finished!");
a?;
b?;
Ok(())
}
async fn release_capture(
@@ -387,209 +479,101 @@ async fn release_capture(
Ok(())
}
fn handle_producer_event(
producer_event: ProducerEvent,
active_clients: &mut Vec<(CaptureHandle, Position)>,
) -> bool {
log::debug!("handling event: {producer_event:?}");
match producer_event {
ProducerEvent::Release => false,
ProducerEvent::Create(c, p) => {
active_clients.push((c, p));
true
}
ProducerEvent::Destroy(c) => {
active_clients.retain(|(h, _)| *h != c);
true
}
}
}
static ALL_CAPABILITIES: &[DeviceCapability] = &[
DeviceCapability::Pointer,
DeviceCapability::PointerAbsolute,
DeviceCapability::Keyboard,
DeviceCapability::Touch,
DeviceCapability::Scroll,
DeviceCapability::Button,
];
async fn handle_ei_event(
ei_event: EiEvent,
current_client: Option<CaptureHandle>,
context: &ei::Context,
event_tx: &Sender<(CaptureHandle, Event)>,
) {
release_session: &Notify,
) -> Result<(), CaptureError> {
match ei_event {
EiEvent::SeatAdded(s) => {
s.seat.bind_capabilities(&[
DeviceCapability::Pointer,
DeviceCapability::PointerAbsolute,
DeviceCapability::Keyboard,
DeviceCapability::Touch,
DeviceCapability::Scroll,
DeviceCapability::Button,
]);
context.flush().unwrap();
s.seat.bind_capabilities(ALL_CAPABILITIES);
context.flush().map_err(|e| io::Error::new(e.kind(), e))?;
}
EiEvent::SeatRemoved(_) => {}
EiEvent::DeviceAdded(_) => {}
EiEvent::DeviceRemoved(_) => {}
EiEvent::DevicePaused(_) => {}
EiEvent::DeviceResumed(_) => {}
EiEvent::KeyboardModifiers(mods) => {
let modifier_event = KeyboardEvent::Modifiers {
mods_depressed: mods.depressed,
mods_latched: mods.latched,
mods_locked: mods.locked,
group: mods.group,
};
if let Some(current_client) = current_client {
event_tx
.send((current_client, Event::Keyboard(modifier_event)))
.await
.unwrap();
}
EiEvent::SeatRemoved(_) | /* EiEvent::DeviceAdded(_) | */ EiEvent::DeviceRemoved(_) => {
log::debug!("releasing session: {ei_event:?}");
release_session.notify_waiters();
}
EiEvent::Frame(_) => {}
EiEvent::DeviceStartEmulating(_) => {
log::debug!("START EMULATING =============>");
}
EiEvent::DeviceStopEmulating(_) => {
log::debug!("==================> STOP EMULATING");
}
EiEvent::PointerMotion(motion) => {
let motion_event = PointerEvent::Motion {
time: motion.time as u32,
dx: motion.dx as f64,
dy: motion.dy as f64,
};
if let Some(current_client) = current_client {
event_tx
.send((current_client, Event::Pointer(motion_event)))
.await
.unwrap();
}
}
EiEvent::PointerMotionAbsolute(_) => {}
EiEvent::Button(button) => {
let button_event = PointerEvent::Button {
time: button.time as u32,
button: button.button,
state: match button.state {
ButtonState::Released => 0,
ButtonState::Press => 1,
},
};
if let Some(current_client) = current_client {
event_tx
.send((current_client, Event::Pointer(button_event)))
.await
.unwrap();
}
}
EiEvent::ScrollDelta(delta) => {
if let Some(handle) = current_client {
let mut events = vec![];
if delta.dy != 0. {
events.push(PointerEvent::Axis {
time: 0,
axis: 0,
value: delta.dy as f64,
});
}
if delta.dx != 0. {
events.push(PointerEvent::Axis {
time: 0,
axis: 1,
value: delta.dx as f64,
});
}
for event in events {
event_tx
.send((handle, Event::Pointer(event)))
.await
.unwrap();
}
}
}
EiEvent::ScrollStop(_) => {}
EiEvent::ScrollCancel(_) => {}
EiEvent::ScrollDiscrete(scroll) => {
if scroll.discrete_dy != 0 {
let event = PointerEvent::AxisDiscrete120 {
axis: 0,
value: scroll.discrete_dy,
};
if let Some(current_client) = current_client {
event_tx
.send((current_client, Event::Pointer(event)))
.await
.unwrap();
}
}
if scroll.discrete_dx != 0 {
let event = PointerEvent::AxisDiscrete120 {
axis: 1,
value: scroll.discrete_dx,
};
if let Some(current_client) = current_client {
event_tx
.send((current_client, Event::Pointer(event)))
.await
.unwrap();
}
};
}
EiEvent::KeyboardKey(key) => {
let key_event = KeyboardEvent::Key {
key: key.key,
state: match key.state {
KeyState::Press => 1,
KeyState::Released => 0,
},
time: key.time as u32,
};
if let Some(current_client) = current_client {
event_tx
.send((current_client, Event::Keyboard(key_event)))
.await
.unwrap();
}
}
EiEvent::TouchDown(_) => {}
EiEvent::TouchUp(_) => {}
EiEvent::TouchMotion(_) => {}
EiEvent::DevicePaused(_) | EiEvent::DeviceResumed(_) => {}
EiEvent::DeviceStartEmulating(_) => log::debug!("START EMULATING"),
EiEvent::DeviceStopEmulating(_) => log::debug!("STOP EMULATING"),
EiEvent::Disconnected(d) => {
log::error!("disconnect: {d:?}");
return Err(CaptureError::Disconnected(format!("{:?}", d.reason)))
}
_ => {
if let Some(handle) = current_client {
for event in Event::from_ei_event(ei_event) {
event_tx.send((handle, event)).await.expect("no channel");
}
}
}
}
Ok(())
}
#[async_trait]
impl<'a> LanMouseInputCapture for LibeiInputCapture<'a> {
async fn create(&mut self, handle: CaptureHandle, pos: Position) -> Result<(), CaptureError> {
let _ = self
.notify_capture
.send(LibeiNotifyEvent::Create(handle, pos))
.await;
Ok(())
}
async fn destroy(&mut self, handle: CaptureHandle) -> Result<(), CaptureError> {
let _ = self
.notify_capture
.send(LibeiNotifyEvent::Destroy(handle))
.await;
Ok(())
}
async fn release(&mut self) -> Result<(), CaptureError> {
self.notify_release.notify_waiters();
Ok(())
}
async fn terminate(&mut self) -> Result<(), CaptureError> {
self.cancellation_token.cancel();
let task = &mut self.capture_task;
log::debug!("waiting for capture to terminate...");
let res = task.await.expect("libei task panic");
log::debug!("done!");
self.terminated = true;
res
}
}
impl<'a> LanMouseInputCapture for LibeiInputCapture<'a> {
fn create(&mut self, handle: super::CaptureHandle, pos: super::Position) -> io::Result<()> {
let notify_tx = self.notify_tx.clone();
tokio::task::spawn_local(async move {
let _ = notify_tx.send(ProducerEvent::Create(handle, pos)).await;
});
Ok(())
}
fn destroy(&mut self, handle: super::CaptureHandle) -> io::Result<()> {
let notify_tx = self.notify_tx.clone();
tokio::task::spawn_local(async move {
let _ = notify_tx.send(ProducerEvent::Destroy(handle)).await;
});
Ok(())
}
fn release(&mut self) -> io::Result<()> {
let notify_tx = self.notify_tx.clone();
tokio::task::spawn_local(async move {
let _ = notify_tx.send(ProducerEvent::Release).await;
});
Ok(())
impl<'a> Drop for LibeiInputCapture<'a> {
fn drop(&mut self) {
if !self.terminated {
/* this workaround is needed until async drop is stabilized */
panic!("LibeiInputCapture dropped without being terminated!");
}
}
}
impl<'a> Stream for LibeiInputCapture<'a> {
type Item = io::Result<(CaptureHandle, Event)>;
type Item = Result<(CaptureHandle, Event), CaptureError>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
match ready!(self.event_rx.poll_recv(cx)) {
None => Poll::Ready(None),
Some(e) => Poll::Ready(Some(Ok(e))),
match self.capture_task.poll_unpin(cx) {
Poll::Ready(r) => match r.expect("failed to join") {
Ok(()) => Poll::Ready(None),
Err(e) => Poll::Ready(Some(Err(e))),
},
Poll::Pending => self.event_rx.poll_recv(cx).map(|e| e.map(Result::Ok)),
}
}
}