Merge remote-tracking branch 'upstream/master'

# Conflicts:
#	Cargo.lock
#	src/server/connection.rs

libs/clipboard/Cargo.toml

@@ -2,16 +2,48 @@
 name = "clipboard"
 version = "0.1.0"
 edition = "2021"
-build= "build.rs"
+build = "build.rs"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [build-dependencies]
 cc = "1.0"
 
+[features]
+default = []
+unix-file-copy-paste = [
+"dep:x11rb",
+"dep:x11-clipboard",
+"dep:rand",
+"dep:fuser",
+"dep:libc",
+"dep:dashmap",
+"dep:percent-encoding",
+"dep:utf16string",
+"dep:once_cell",
+"dep:cacao"
+]
+
 [dependencies]
 thiserror = "1.0"
 lazy_static = "1.4"
 serde = "1.0"
 serde_derive = "1.0"
 hbb_common = { path = "../hbb_common" }
+parking_lot = {version = "0.12"}
+
+[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies]
+rand = {version = "0.8", optional = true}
+fuser = {version = "0.13", optional = true}
+libc = {version = "0.2", optional = true}
+dashmap = {version ="5.5", optional = true}
+utf16string = {version = "0.2", optional = true}
+once_cell = {version = "1.18", optional = true}
+
+[target.'cfg(target_os = "linux")'.dependencies]
+percent-encoding = {version  ="2.3", optional = true}
+x11-clipboard = {git="https://github.com/clslaid/x11-clipboard", branch = "feat/store-batch", optional = true}
+x11rb =  {version = "0.12", features = ["all-extensions"], optional = true}
+
+[target.'cfg(target_os = "macos")'.dependencies]
+cacao = {git="https://github.com/clslaid/cacao", branch = "feat/set-file-urls", optional = true}

libs/clipboard/README.md

@@ -3,8 +3,74 @@
 Copy files and text through network.
 Main lowlevel logic from [FreeRDP](https://github.com/FreeRDP/FreeRDP).
 
+To enjoy file copy and paste feature on Linux/OSX,
+please build with `unix-file-copy-paste` feature.
+
 TODO: Move this lib to a separate project.
 
+## How it works
+
+Terminalogies:
+
+- cliprdr: this module
+- local: the endpoint which initiates a file copy events
+- remote: the endpoint which paste the file copied from `local`
+
+The main algorithm of copying and pasting files is from
+[Remote Desktop Protocol: Clipboard Virtual Channel Extension](https://winprotocoldoc.blob.core.windows.net/productionwindowsarchives/MS-RDPECLIP/%5bMS-RDPECLIP%5d.pdf),
+and could be concluded as:
+
+0. local and remote notify each other that it's ready.
+1. local subscribes/listening to the system's clipboard for file copy
+2. local once got file copy event, notice the remote
+3. remote confirms receive and try pulls the file list
+4. local updates its file-list, the remote flushes pulled file list to the clipboard
+5. remote OS or desktop manager initiates a paste, making other programs reading
+   clipboard files. Convert those reading requests to RPCs
+
+   - on Windows, all file reading will go through the stream file API
+   - on Linux/OSX, FUSE is used for converting reading requests to RPCs
+     - in case of local clipboard been transferred back
+       and leading to a dead loop,
+       all file copy event pointing at the FUSE directory will be ignored
+
+6. finishing pasting all files one by one.
+
+In a perspective of network data transferring:
+
+```mermaid
+sequenceDiagram
+    participant l as Local
+    participant r as Remote
+    note over l, r: Initialize
+    l ->> r: Monitor Ready
+    r ->> l: Monitor Ready
+    loop Get clipboard update
+        l ->> r: Format List (I got update)
+        r ->> l: Format List Response (notified)
+        r ->> l: Format Data Request (requests file list)
+        activate l
+            note left of l: Retrive file list from system clipboard
+            l ->> r: Format Data Response (containing file list)
+        deactivate l
+        note over r: Update system clipboard with received file list
+    end
+    loop Some application requests copied files
+        note right of r: application reads file from x to x+y
+        note over r: the file is the a-th file on list
+        r ->> l: File Contents Request (read file a offset x size y)
+        activate l
+            note left of l: Find a-th file on list, read from x to x+y
+            l ->> r: File Contents Response (contents of file a offset x size y)
+        deactivate l
+    end
+```
+
+Note: In actual implementation, both sides could play send clipboard update
+and request file contents.
+There is no such limitation that only local can update clipboard
+and copy files to remote.
+
 ## impl
 
 ### windows
@@ -14,3 +80,82 @@ TODO: Move this lib to a separate project.
 ![A1->B1](./docs/assets/win_A_B.png)
 
 ![B1->A1](./docs/assets/win_B_A.png)
+
+The protocol was originally designed as an extension of the Windows RDP,
+so the specific message packages fits windows well.
+
+When starting cliprdr, a thread is spawn to create a invisible window
+and to subscribe to OLE clipboard events.
+The window's callback (see `cliprdr_proc` in `src/windows/wf_cliprdr.c`) was
+set to handle a variaty of events.
+
+Detailed implementation is shown in pictures above.
+
+### Linux/OSX
+
+The Cliprdr Server implementation has mainly 3 parts:
+
+- Clipboard Client
+- Local File list
+- FUSE server
+
+#### Clipboard Client
+
+The clipboard client has a thread polling for file urls on clipboard.
+
+If the client found any updates of file urls,
+after filtering out those pointing to our FUSE directory or duplicated,
+send format list directly to remote.
+
+The cliprdr server also uses clipboard client for setting clipboard,
+or retrive paths from system.
+
+#### Local File List
+
+The local file list is a temperary list of file metadata.
+When receiving file contents PDU from peer, the server picks
+out the file requested and open it for reading if necessary.
+
+Also when receiving Format Data Request PDU from remote asking for file list,
+the local file list should be rebuilt from file list retrieved from Clipboard Client.
+
+Some caching and preloading could done on it since applications are likely to read
+on the list sequentially.
+
+#### FUSE server
+
+The FUSE server could convert POSIX file reading request to File Contents
+Request/Response RPCs.
+
+When received file list from remote,
+the FUSE server will figure out the file system tree and rearrange its content.
+
+#### Groceries
+
+- The protocol was originally implemented for windows,
+  so paths in PDU will all be converted to DOS formats in UTF-16 LE encoding,
+  and datetimes will be converted to LDAP timestamp instead of
+  unix timestamp
+
+  ```text
+  UNIX
+  /usr/bin/rustdesk
+  ->
+  DOS
+  \usr\bin\rustdesk
+  ```
+
+- To better fit for preserving permissions on unix-like platforms,
+  a reserved area of FileDescriptor PDU
+
+- you may notice
+  the mountpoint is still occupied after the application quits.
+  That's because the FUSE server was not mounted with `AUTO_UNMOUNT`.
+  - It's hard to implement gressful shutdown for a multi-processed program
+  - `AUTO_UNMOUNT` was not enabled by default and requires enable
+    `user_allow_other` in configure. Letting users edit such global
+    configuration to use this feature might not be a good idea.
+  - use [`umount()`](https://man7.org/linux/man-pages/man2/umount.2.html)
+    syscall to unmount will also require that option.
+  - we currently directly call [`umount`](https://man7.org/linux/man-pages/man8/umount.8.html)
+    program to unmount dangling FUSE server. It worked perfectly for now.

libs/clipboard/build.rs

@@ -1,43 +1,35 @@
-use cc;
-
+#[cfg(target_os = "windows")]
 fn build_c_impl() {
     let mut build = cc::Build::new();
 
-    #[cfg(target_os = "windows")]
     build.file("src/windows/wf_cliprdr.c");
-    #[cfg(target_os = "linux")]
-    build.file("src/X11/xf_cliprdr.c");
-    #[cfg(target_os = "macos")]
-    build.file("src/OSX/Clipboard.m");
 
-    build.flag_if_supported("-Wno-c++0x-extensions");
-    build.flag_if_supported("-Wno-return-type-c-linkage");
-    build.flag_if_supported("-Wno-invalid-offsetof");
-    build.flag_if_supported("-Wno-unused-parameter");
+    {
+        build.flag_if_supported("-Wno-c++0x-extensions");
+        build.flag_if_supported("-Wno-return-type-c-linkage");
+        build.flag_if_supported("-Wno-invalid-offsetof");
+        build.flag_if_supported("-Wno-unused-parameter");
 
-    if build.get_compiler().is_like_msvc() {
-        build.define("WIN32", "");
-        // build.define("_AMD64_", "");
-        build.flag("-Z7");
-        build.flag("-GR-");
-        // build.flag("-std:c++11");
-    } else {
-        build.flag("-fPIC");
-        // build.flag("-std=c++11");
-        // build.flag("-include");
-        // build.flag(&confdefs_path.to_string_lossy());
+        if build.get_compiler().is_like_msvc() {
+            build.define("WIN32", "");
+            // build.define("_AMD64_", "");
+            build.flag("-Z7");
+            build.flag("-GR-");
+            // build.flag("-std:c++11");
+        } else {
+            build.flag("-fPIC");
+            // build.flag("-std=c++11");
+            // build.flag("-include");
+            // build.flag(&confdefs_path.to_string_lossy());
+        }
+
+        build.compile("mycliprdr");
     }
 
-    build.compile("mycliprdr");
-
-    #[cfg(target_os = "windows")]
     println!("cargo:rerun-if-changed=src/windows/wf_cliprdr.c");
-    #[cfg(target_os = "linux")]
-    println!("cargo:rerun-if-changed=src/X11/xf_cliprdr.c");
-    #[cfg(target_os = "macos")]
-    println!("cargo:rerun-if-changed=src/OSX/Clipboard.m");
 }
 
 fn main() {
+    #[cfg(target_os = "windows")]
     build_c_impl();
 }

libs/clipboard/src/OSX/Clipboard.m

@@ -1,11 +0,0 @@
-#include "../cliprdr.h"
-
-void mac_cliprdr_init(CliprdrClientContext *cliprdr)
-{
-    (void)cliprdr;
-}
-
-void mac_cliprdr_uninit(CliprdrClientContext *cliprdr)
-{
-    (void)cliprdr;
-}

libs/clipboard/src/X11/xf_cliprdr.c

@@ -1,11 +0,0 @@
-#include "../cliprdr.h"
-
-void xf_cliprdr_init(CliprdrClientContext* cliprdr)
-{
-    (void)cliprdr;
-}
-
-void xf_cliprdr_uninit( CliprdrClientContext* cliprdr)
-{
-    (void)cliprdr;
-}

libs/clipboard/src/cliprdr.rs

@@ -1,573 +0,0 @@
-#![allow(dead_code)]
-#![allow(non_camel_case_types)]
-#![allow(unused_variables)]
-#![allow(non_snake_case)]
-#![allow(deref_nullptr)]
-
-use std::{boxed::Box, result::Result};
-use thiserror::Error;
-
-pub type size_t = ::std::os::raw::c_ulonglong;
-pub type __vcrt_bool = bool;
-pub type wchar_t = ::std::os::raw::c_ushort;
-
-pub type POINTER_64_INT = ::std::os::raw::c_ulonglong;
-pub type INT8 = ::std::os::raw::c_schar;
-pub type PINT8 = *mut ::std::os::raw::c_schar;
-pub type INT16 = ::std::os::raw::c_short;
-pub type PINT16 = *mut ::std::os::raw::c_short;
-pub type INT32 = ::std::os::raw::c_int;
-pub type PINT32 = *mut ::std::os::raw::c_int;
-pub type INT64 = ::std::os::raw::c_longlong;
-pub type PINT64 = *mut ::std::os::raw::c_longlong;
-pub type UINT8 = ::std::os::raw::c_uchar;
-pub type PUINT8 = *mut ::std::os::raw::c_uchar;
-pub type UINT16 = ::std::os::raw::c_ushort;
-pub type PUINT16 = *mut ::std::os::raw::c_ushort;
-pub type UINT32 = ::std::os::raw::c_uint;
-pub type PUINT32 = *mut ::std::os::raw::c_uint;
-pub type UINT64 = ::std::os::raw::c_ulonglong;
-pub type PUINT64 = *mut ::std::os::raw::c_ulonglong;
-pub type LONG32 = ::std::os::raw::c_int;
-pub type PLONG32 = *mut ::std::os::raw::c_int;
-pub type ULONG32 = ::std::os::raw::c_uint;
-pub type PULONG32 = *mut ::std::os::raw::c_uint;
-pub type DWORD32 = ::std::os::raw::c_uint;
-pub type PDWORD32 = *mut ::std::os::raw::c_uint;
-pub type INT_PTR = ::std::os::raw::c_longlong;
-pub type PINT_PTR = *mut ::std::os::raw::c_longlong;
-pub type UINT_PTR = ::std::os::raw::c_ulonglong;
-pub type PUINT_PTR = *mut ::std::os::raw::c_ulonglong;
-pub type LONG_PTR = ::std::os::raw::c_longlong;
-pub type PLONG_PTR = *mut ::std::os::raw::c_longlong;
-pub type ULONG_PTR = ::std::os::raw::c_ulonglong;
-pub type PULONG_PTR = *mut ::std::os::raw::c_ulonglong;
-pub type SHANDLE_PTR = ::std::os::raw::c_longlong;
-pub type HANDLE_PTR = ::std::os::raw::c_ulonglong;
-pub type UHALF_PTR = ::std::os::raw::c_uint;
-pub type PUHALF_PTR = *mut ::std::os::raw::c_uint;
-pub type HALF_PTR = ::std::os::raw::c_int;
-pub type PHALF_PTR = *mut ::std::os::raw::c_int;
-pub type SIZE_T = ULONG_PTR;
-pub type PSIZE_T = *mut ULONG_PTR;
-pub type SSIZE_T = LONG_PTR;
-pub type PSSIZE_T = *mut LONG_PTR;
-pub type DWORD_PTR = ULONG_PTR;
-pub type PDWORD_PTR = *mut ULONG_PTR;
-pub type LONG64 = ::std::os::raw::c_longlong;
-pub type PLONG64 = *mut ::std::os::raw::c_longlong;
-pub type ULONG64 = ::std::os::raw::c_ulonglong;
-pub type PULONG64 = *mut ::std::os::raw::c_ulonglong;
-pub type DWORD64 = ::std::os::raw::c_ulonglong;
-pub type PDWORD64 = *mut ::std::os::raw::c_ulonglong;
-pub type KAFFINITY = ULONG_PTR;
-pub type PKAFFINITY = *mut KAFFINITY;
-pub type PVOID = *mut ::std::os::raw::c_void;
-pub type CHAR = ::std::os::raw::c_char;
-pub type SHORT = ::std::os::raw::c_short;
-pub type LONG = ::std::os::raw::c_long;
-pub type WCHAR = wchar_t;
-pub type PWCHAR = *mut WCHAR;
-pub type LPWCH = *mut WCHAR;
-pub type PWCH = *mut WCHAR;
-pub type LPCWCH = *const WCHAR;
-pub type PCWCH = *const WCHAR;
-pub type NWPSTR = *mut WCHAR;
-pub type LPWSTR = *mut WCHAR;
-pub type PWSTR = *mut WCHAR;
-pub type PZPWSTR = *mut PWSTR;
-pub type PCZPWSTR = *const PWSTR;
-pub type LPUWSTR = *mut WCHAR;
-pub type PUWSTR = *mut WCHAR;
-pub type LPCWSTR = *const WCHAR;
-pub type PCWSTR = *const WCHAR;
-pub type PZPCWSTR = *mut PCWSTR;
-pub type PCZPCWSTR = *const PCWSTR;
-pub type LPCUWSTR = *const WCHAR;
-pub type PCUWSTR = *const WCHAR;
-pub type PZZWSTR = *mut WCHAR;
-pub type PCZZWSTR = *const WCHAR;
-pub type PUZZWSTR = *mut WCHAR;
-pub type PCUZZWSTR = *const WCHAR;
-pub type PNZWCH = *mut WCHAR;
-pub type PCNZWCH = *const WCHAR;
-pub type PUNZWCH = *mut WCHAR;
-pub type PCUNZWCH = *const WCHAR;
-pub type PCHAR = *mut CHAR;
-pub type LPCH = *mut CHAR;
-pub type PCH = *mut CHAR;
-pub type LPCCH = *const CHAR;
-pub type PCCH = *const CHAR;
-pub type NPSTR = *mut CHAR;
-pub type LPSTR = *mut CHAR;
-pub type PSTR = *mut CHAR;
-pub type PZPSTR = *mut PSTR;
-pub type PCZPSTR = *const PSTR;
-pub type LPCSTR = *const CHAR;
-pub type PCSTR = *const CHAR;
-pub type PZPCSTR = *mut PCSTR;
-pub type PCZPCSTR = *const PCSTR;
-pub type PZZSTR = *mut CHAR;
-pub type PCZZSTR = *const CHAR;
-pub type PNZCH = *mut CHAR;
-pub type PCNZCH = *const CHAR;
-pub type TCHAR = ::std::os::raw::c_char;
-pub type PTCHAR = *mut ::std::os::raw::c_char;
-pub type TBYTE = ::std::os::raw::c_uchar;
-pub type PTBYTE = *mut ::std::os::raw::c_uchar;
-pub type LPTCH = LPCH;
-pub type PTCH = LPCH;
-pub type LPCTCH = LPCCH;
-pub type PCTCH = LPCCH;
-pub type PTSTR = LPSTR;
-pub type LPTSTR = LPSTR;
-pub type PUTSTR = LPSTR;
-pub type LPUTSTR = LPSTR;
-pub type PCTSTR = LPCSTR;
-pub type LPCTSTR = LPCSTR;
-pub type PCUTSTR = LPCSTR;
-pub type LPCUTSTR = LPCSTR;
-pub type PZZTSTR = PZZSTR;
-pub type PUZZTSTR = PZZSTR;
-pub type PCZZTSTR = PCZZSTR;
-pub type PCUZZTSTR = PCZZSTR;
-pub type PZPTSTR = PZPSTR;
-pub type PNZTCH = PNZCH;
-pub type PUNZTCH = PNZCH;
-pub type PCNZTCH = PCNZCH;
-pub type PCUNZTCH = PCNZCH;
-pub type PSHORT = *mut SHORT;
-pub type PLONG = *mut LONG;
-pub type ULONG = ::std::os::raw::c_ulong;
-pub type PULONG = *mut ULONG;
-pub type USHORT = ::std::os::raw::c_ushort;
-pub type PUSHORT = *mut USHORT;
-pub type UCHAR = ::std::os::raw::c_uchar;
-pub type PUCHAR = *mut UCHAR;
-pub type PSZ = *mut ::std::os::raw::c_char;
-pub type DWORD = ::std::os::raw::c_ulong;
-pub type BOOL = ::std::os::raw::c_int;
-pub type BYTE = ::std::os::raw::c_uchar;
-pub type WORD = ::std::os::raw::c_ushort;
-pub type FLOAT = f32;
-pub type PFLOAT = *mut FLOAT;
-pub type PBOOL = *mut BOOL;
-pub type LPBOOL = *mut BOOL;
-pub type PBYTE = *mut BYTE;
-pub type LPBYTE = *mut BYTE;
-pub type PINT = *mut ::std::os::raw::c_int;
-pub type LPINT = *mut ::std::os::raw::c_int;
-pub type PWORD = *mut WORD;
-pub type LPWORD = *mut WORD;
-pub type LPLONG = *mut ::std::os::raw::c_long;
-pub type PDWORD = *mut DWORD;
-pub type LPDWORD = *mut DWORD;
-pub type LPVOID = *mut ::std::os::raw::c_void;
-pub type LPCVOID = *const ::std::os::raw::c_void;
-pub type INT = ::std::os::raw::c_int;
-pub type UINT = ::std::os::raw::c_uint;
-pub type PUINT = *mut ::std::os::raw::c_uint;
-pub type va_list = *mut ::std::os::raw::c_char;
-
-pub const TRUE: ::std::os::raw::c_int = 1;
-pub const FALSE: ::std::os::raw::c_int = 0;
-
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct _CLIPRDR_HEADER {
-    pub connID: UINT32,
-    pub msgType: UINT16,
-    pub msgFlags: UINT16,
-    pub dataLen: UINT32,
-}
-pub type CLIPRDR_HEADER = _CLIPRDR_HEADER;
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct _CLIPRDR_CAPABILITY_SET {
-    pub capabilitySetType: UINT16,
-    pub capabilitySetLength: UINT16,
-}
-pub type CLIPRDR_CAPABILITY_SET = _CLIPRDR_CAPABILITY_SET;
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct _CLIPRDR_GENERAL_CAPABILITY_SET {
-    pub capabilitySetType: UINT16,
-    pub capabilitySetLength: UINT16,
-    pub version: UINT32,
-    pub generalFlags: UINT32,
-}
-pub type CLIPRDR_GENERAL_CAPABILITY_SET = _CLIPRDR_GENERAL_CAPABILITY_SET;
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct _CLIPRDR_CAPABILITIES {
-    pub connID: UINT32,
-    pub msgType: UINT16,
-    pub msgFlags: UINT16,
-    pub dataLen: UINT32,
-    pub cCapabilitiesSets: UINT32,
-    pub capabilitySets: *mut CLIPRDR_CAPABILITY_SET,
-}
-pub type CLIPRDR_CAPABILITIES = _CLIPRDR_CAPABILITIES;
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct _CLIPRDR_MONITOR_READY {
-    pub connID: UINT32,
-    pub msgType: UINT16,
-    pub msgFlags: UINT16,
-    pub dataLen: UINT32,
-}
-pub type CLIPRDR_MONITOR_READY = _CLIPRDR_MONITOR_READY;
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct _CLIPRDR_TEMP_DIRECTORY {
-    pub connID: UINT32,
-    pub msgType: UINT16,
-    pub msgFlags: UINT16,
-    pub dataLen: UINT32,
-    pub szTempDir: [::std::os::raw::c_char; 520usize],
-}
-pub type CLIPRDR_TEMP_DIRECTORY = _CLIPRDR_TEMP_DIRECTORY;
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct _CLIPRDR_FORMAT {
-    pub formatId: UINT32,
-    pub formatName: *mut ::std::os::raw::c_char,
-}
-pub type CLIPRDR_FORMAT = _CLIPRDR_FORMAT;
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct _CLIPRDR_FORMAT_LIST {
-    pub connID: UINT32,
-    pub msgType: UINT16,
-    pub msgFlags: UINT16,
-    pub dataLen: UINT32,
-    pub numFormats: UINT32,
-    pub formats: *mut CLIPRDR_FORMAT,
-}
-pub type CLIPRDR_FORMAT_LIST = _CLIPRDR_FORMAT_LIST;
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct _CLIPRDR_FORMAT_LIST_RESPONSE {
-    pub connID: UINT32,
-    pub msgType: UINT16,
-    pub msgFlags: UINT16,
-    pub dataLen: UINT32,
-}
-pub type CLIPRDR_FORMAT_LIST_RESPONSE = _CLIPRDR_FORMAT_LIST_RESPONSE;
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct _CLIPRDR_LOCK_CLIPBOARD_DATA {
-    pub connID: UINT32,
-    pub msgType: UINT16,
-    pub msgFlags: UINT16,
-    pub dataLen: UINT32,
-    pub clipDataId: UINT32,
-}
-pub type CLIPRDR_LOCK_CLIPBOARD_DATA = _CLIPRDR_LOCK_CLIPBOARD_DATA;
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct _CLIPRDR_UNLOCK_CLIPBOARD_DATA {
-    pub connID: UINT32,
-    pub msgType: UINT16,
-    pub msgFlags: UINT16,
-    pub dataLen: UINT32,
-    pub clipDataId: UINT32,
-}
-pub type CLIPRDR_UNLOCK_CLIPBOARD_DATA = _CLIPRDR_UNLOCK_CLIPBOARD_DATA;
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct _CLIPRDR_FORMAT_DATA_REQUEST {
-    pub connID: UINT32,
-    pub msgType: UINT16,
-    pub msgFlags: UINT16,
-    pub dataLen: UINT32,
-    pub requestedFormatId: UINT32,
-}
-pub type CLIPRDR_FORMAT_DATA_REQUEST = _CLIPRDR_FORMAT_DATA_REQUEST;
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct _CLIPRDR_FORMAT_DATA_RESPONSE {
-    pub connID: UINT32,
-    pub msgType: UINT16,
-    pub msgFlags: UINT16,
-    pub dataLen: UINT32,
-    pub requestedFormatData: *const BYTE,
-}
-pub type CLIPRDR_FORMAT_DATA_RESPONSE = _CLIPRDR_FORMAT_DATA_RESPONSE;
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct _CLIPRDR_FILE_CONTENTS_REQUEST {
-    pub connID: UINT32,
-    pub msgType: UINT16,
-    pub msgFlags: UINT16,
-    pub dataLen: UINT32,
-    pub streamId: UINT32,
-    pub listIndex: UINT32,
-    pub dwFlags: UINT32,
-    pub nPositionLow: UINT32,
-    pub nPositionHigh: UINT32,
-    pub cbRequested: UINT32,
-    pub haveClipDataId: BOOL,
-    pub clipDataId: UINT32,
-}
-pub type CLIPRDR_FILE_CONTENTS_REQUEST = _CLIPRDR_FILE_CONTENTS_REQUEST;
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct _CLIPRDR_FILE_CONTENTS_RESPONSE {
-    pub connID: UINT32,
-    pub msgType: UINT16,
-    pub msgFlags: UINT16,
-    pub dataLen: UINT32,
-    pub streamId: UINT32,
-    pub cbRequested: UINT32,
-    pub requestedData: *const BYTE,
-}
-pub type CLIPRDR_FILE_CONTENTS_RESPONSE = _CLIPRDR_FILE_CONTENTS_RESPONSE;
-pub type CliprdrClientContext = _cliprdr_client_context;
-#[repr(C)]
-#[derive(Debug, Copy, Clone)]
-pub struct _NOTIFICATION_MESSAGE {
-    pub r#type: UINT32, // 0 - info, 1 - warning, 2 - error
-    pub msg: *const BYTE,
-    pub details: *const BYTE,
-}
-pub type NOTIFICATION_MESSAGE = _NOTIFICATION_MESSAGE;
-pub type pcCliprdrServerCapabilities = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        capabilities: *const CLIPRDR_CAPABILITIES,
-    ) -> UINT,
->;
-pub type pcCliprdrClientCapabilities = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        capabilities: *const CLIPRDR_CAPABILITIES,
-    ) -> UINT,
->;
-pub type pcCliprdrMonitorReady = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        monitorReady: *const CLIPRDR_MONITOR_READY,
-    ) -> UINT,
->;
-pub type pcCliprdrTempDirectory = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        tempDirectory: *const CLIPRDR_TEMP_DIRECTORY,
-    ) -> UINT,
->;
-pub type pcNotifyClipboardMsg =
-    ::std::option::Option<unsafe extern "C" fn(connID: UINT32, msg: *const NOTIFICATION_MESSAGE) -> UINT>;
-pub type pcCliprdrClientFormatList = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        formatList: *const CLIPRDR_FORMAT_LIST,
-    ) -> UINT,
->;
-pub type pcCliprdrServerFormatList = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        formatList: *const CLIPRDR_FORMAT_LIST,
-    ) -> UINT,
->;
-pub type pcCliprdrClientFormatListResponse = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        formatListResponse: *const CLIPRDR_FORMAT_LIST_RESPONSE,
-    ) -> UINT,
->;
-pub type pcCliprdrServerFormatListResponse = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        formatListResponse: *const CLIPRDR_FORMAT_LIST_RESPONSE,
-    ) -> UINT,
->;
-pub type pcCliprdrClientLockClipboardData = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        lockClipboardData: *const CLIPRDR_LOCK_CLIPBOARD_DATA,
-    ) -> UINT,
->;
-pub type pcCliprdrServerLockClipboardData = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        lockClipboardData: *const CLIPRDR_LOCK_CLIPBOARD_DATA,
-    ) -> UINT,
->;
-pub type pcCliprdrClientUnlockClipboardData = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        unlockClipboardData: *const CLIPRDR_UNLOCK_CLIPBOARD_DATA,
-    ) -> UINT,
->;
-pub type pcCliprdrServerUnlockClipboardData = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        unlockClipboardData: *const CLIPRDR_UNLOCK_CLIPBOARD_DATA,
-    ) -> UINT,
->;
-pub type pcCliprdrClientFormatDataRequest = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        formatDataRequest: *const CLIPRDR_FORMAT_DATA_REQUEST,
-    ) -> UINT,
->;
-pub type pcCliprdrServerFormatDataRequest = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        formatDataRequest: *const CLIPRDR_FORMAT_DATA_REQUEST,
-    ) -> UINT,
->;
-pub type pcCliprdrClientFormatDataResponse = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        formatDataResponse: *const CLIPRDR_FORMAT_DATA_RESPONSE,
-    ) -> UINT,
->;
-pub type pcCliprdrServerFormatDataResponse = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        formatDataResponse: *const CLIPRDR_FORMAT_DATA_RESPONSE,
-    ) -> UINT,
->;
-pub type pcCliprdrClientFileContentsRequest = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        fileContentsRequest: *const CLIPRDR_FILE_CONTENTS_REQUEST,
-    ) -> UINT,
->;
-pub type pcCliprdrServerFileContentsRequest = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        fileContentsRequest: *const CLIPRDR_FILE_CONTENTS_REQUEST,
-    ) -> UINT,
->;
-pub type pcCliprdrClientFileContentsResponse = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        fileContentsResponse: *const CLIPRDR_FILE_CONTENTS_RESPONSE,
-    ) -> UINT,
->;
-pub type pcCliprdrServerFileContentsResponse = ::std::option::Option<
-    unsafe extern "C" fn(
-        context: *mut CliprdrClientContext,
-        fileContentsResponse: *const CLIPRDR_FILE_CONTENTS_RESPONSE,
-    ) -> UINT,
->;
-
-// TODO: hide more members of clipboard context
-#[repr(C)]
-#[derive(Debug, Clone)]
-pub struct _cliprdr_client_context {
-    pub Custom: *mut ::std::os::raw::c_void,
-    pub EnableFiles: BOOL,
-    pub EnableOthers: BOOL,
-    pub IsStopped: BOOL,
-    pub ResponseWaitTimeoutSecs: UINT32,
-    pub ServerCapabilities: pcCliprdrServerCapabilities,
-    pub ClientCapabilities: pcCliprdrClientCapabilities,
-    pub MonitorReady: pcCliprdrMonitorReady,
-    pub TempDirectory: pcCliprdrTempDirectory,
-    pub NotifyClipboardMsg: pcNotifyClipboardMsg,
-    pub ClientFormatList: pcCliprdrClientFormatList,
-    pub ServerFormatList: pcCliprdrServerFormatList,
-    pub ClientFormatListResponse: pcCliprdrClientFormatListResponse,
-    pub ServerFormatListResponse: pcCliprdrServerFormatListResponse,
-    pub ClientLockClipboardData: pcCliprdrClientLockClipboardData,
-    pub ServerLockClipboardData: pcCliprdrServerLockClipboardData,
-    pub ClientUnlockClipboardData: pcCliprdrClientUnlockClipboardData,
-    pub ServerUnlockClipboardData: pcCliprdrServerUnlockClipboardData,
-    pub ClientFormatDataRequest: pcCliprdrClientFormatDataRequest,
-    pub ServerFormatDataRequest: pcCliprdrServerFormatDataRequest,
-    pub ClientFormatDataResponse: pcCliprdrClientFormatDataResponse,
-    pub ServerFormatDataResponse: pcCliprdrServerFormatDataResponse,
-    pub ClientFileContentsRequest: pcCliprdrClientFileContentsRequest,
-    pub ServerFileContentsRequest: pcCliprdrServerFileContentsRequest,
-    pub ClientFileContentsResponse: pcCliprdrClientFileContentsResponse,
-    pub ServerFileContentsResponse: pcCliprdrServerFileContentsResponse,
-    pub LastRequestedFormatId: UINT32,
-}
-
-// #[link(name = "user32")]
-// #[link(name = "ole32")]
-extern "C" {
-    pub(crate) fn init_cliprdr(context: *mut CliprdrClientContext) -> BOOL;
-    pub(crate) fn uninit_cliprdr(context: *mut CliprdrClientContext) -> BOOL;
-    pub(crate) fn empty_cliprdr(context: *mut CliprdrClientContext, connID: UINT32) -> BOOL;
-}
-
-#[derive(Error, Debug)]
-pub enum CliprdrError {
-    #[error("invalid cliprdr name")]
-    CliprdrName,
-    #[error("failed to init cliprdr")]
-    CliprdrInit,
-    #[error("unknown cliprdr error")]
-    Unknown,
-}
-
-impl CliprdrClientContext {
-    pub fn create(
-        enable_files: bool,
-        enable_others: bool,
-        response_wait_timeout_secs: u32,
-        notify_callback: pcNotifyClipboardMsg,
-        client_format_list: pcCliprdrClientFormatList,
-        client_format_list_response: pcCliprdrClientFormatListResponse,
-        client_format_data_request: pcCliprdrClientFormatDataRequest,
-        client_format_data_response: pcCliprdrClientFormatDataResponse,
-        client_file_contents_request: pcCliprdrClientFileContentsRequest,
-        client_file_contents_response: pcCliprdrClientFileContentsResponse,
-    ) -> Result<Box<Self>, CliprdrError> {
-        let context = CliprdrClientContext {
-            Custom: 0 as *mut _,
-            EnableFiles: if enable_files { TRUE } else { FALSE },
-            EnableOthers: if enable_others { TRUE } else { FALSE },
-            IsStopped: FALSE,
-            ResponseWaitTimeoutSecs: response_wait_timeout_secs,
-            ServerCapabilities: None,
-            ClientCapabilities: None,
-            MonitorReady: None,
-            TempDirectory: None,
-            NotifyClipboardMsg: notify_callback,
-            ClientFormatList: client_format_list,
-            ServerFormatList: None,
-            ClientFormatListResponse: client_format_list_response,
-            ServerFormatListResponse: None,
-            ClientLockClipboardData: None,
-            ServerLockClipboardData: None,
-            ClientUnlockClipboardData: None,
-            ServerUnlockClipboardData: None,
-            ClientFormatDataRequest: client_format_data_request,
-            ServerFormatDataRequest: None,
-            ClientFormatDataResponse: client_format_data_response,
-            ServerFormatDataResponse: None,
-            ClientFileContentsRequest: client_file_contents_request,
-            ServerFileContentsRequest: None,
-            ClientFileContentsResponse: client_file_contents_response,
-            ServerFileContentsResponse: None,
-            LastRequestedFormatId: 0,
-        };
-        let mut context = Box::new(context);
-        unsafe {
-            if FALSE == init_cliprdr(&mut (*context)) {
-                println!("Failed to init cliprdr");
-                Err(CliprdrError::CliprdrInit)
-            } else {
-                Ok(context)
-            }
-        }
-    }
-}
-
-impl Drop for CliprdrClientContext {
-    fn drop(&mut self) {
-        unsafe {
-            if FALSE == uninit_cliprdr(&mut *self) {
-                println!("Failed to uninit cliprdr");
-            } else {
-                println!("Succeeded to uninit cliprdr");
-            }
-        }
-    }
-}

libs/clipboard/src/context_send.rs

@@ -1,68 +1,72 @@
-use crate::cliprdr::*;
-use hbb_common::log;
+use hbb_common::{log, ResultType};
 use std::sync::Mutex;
 
+use crate::CliprdrServiceContext;
+
 const CLIPBOARD_RESPONSE_WAIT_TIMEOUT_SECS: u32 = 30;
 
 lazy_static::lazy_static! {
-    static ref CONTEXT_SEND: ContextSend = ContextSend{addr: Mutex::new(0)};
+    static ref CONTEXT_SEND: ContextSend = ContextSend{addr: Mutex::new(None)};
 }
 
 pub struct ContextSend {
-    addr: Mutex<u64>,
+    addr: Mutex<Option<Box<dyn CliprdrServiceContext>>>,
 }
 
 impl ContextSend {
     #[inline]
     pub fn is_enabled() -> bool {
-        *CONTEXT_SEND.addr.lock().unwrap() != 0
+        CONTEXT_SEND.addr.lock().unwrap().is_some()
     }
 
     pub fn set_is_stopped() {
-        let _res = Self::proc(|c| {
-            c.IsStopped = TRUE;
-            0
-        });
+        let _res = Self::proc(|c| c.set_is_stopped().map_err(|e| e.into()));
     }
 
     pub fn enable(enabled: bool) {
         let mut lock = CONTEXT_SEND.addr.lock().unwrap();
         if enabled {
-            if *lock == 0 {
-                match crate::create_cliprdr_context(
-                    true,
-                    false,
-                    CLIPBOARD_RESPONSE_WAIT_TIMEOUT_SECS,
-                ) {
-                    Ok(context) => {
-                        log::info!("clipboard context for file transfer created.");
-                        *lock = Box::into_raw(context) as _;
-                    }
-                    Err(err) => {
-                        log::error!(
-                            "Create clipboard context for file transfer: {}",
-                            err.to_string()
-                        );
-                    }
+            if lock.is_some() {
+                return;
+            }
+            match crate::create_cliprdr_context(true, false, CLIPBOARD_RESPONSE_WAIT_TIMEOUT_SECS) {
+                Ok(context) => {
+                    log::info!("clipboard context for file transfer created.");
+                    *lock = Some(context)
+                }
+                Err(err) => {
+                    log::error!(
+                        "create clipboard context for file transfer: {}",
+                        err.to_string()
+                    );
                 }
             }
-        } else {
-            if *lock != 0 {
-                unsafe {
-                    let _ = Box::from_raw(*lock as *mut CliprdrClientContext);
-                }
-                log::info!("clipboard context for file transfer destroyed.");
-                *lock = 0;
-            }
+        } else if let Some(_clp) = lock.take() {
+            *lock = None;
+            log::info!("clipboard context for file transfer destroyed.");
         }
     }
 
-    pub fn proc<F: FnOnce(&mut CliprdrClientContext) -> u32>(f: F) -> u32 {
-        let lock = CONTEXT_SEND.addr.lock().unwrap();
-        if *lock != 0 {
-            unsafe { f(&mut *(*lock as *mut CliprdrClientContext)) }
-        } else {
-            0
+    /// make sure the clipboard context is enabled.
+    pub fn make_sure_enabled() -> ResultType<()> {
+        let mut lock = CONTEXT_SEND.addr.lock().unwrap();
+        if lock.is_some() {
+            return Ok(());
+        }
+
+        let ctx = crate::create_cliprdr_context(true, false, CLIPBOARD_RESPONSE_WAIT_TIMEOUT_SECS)?;
+        *lock = Some(ctx);
+        log::info!("clipboard context for file transfer recreated.");
+        Ok(())
+    }
+
+    pub fn proc<F: FnOnce(&mut Box<dyn CliprdrServiceContext>) -> ResultType<()>>(
+        f: F,
+    ) -> ResultType<()> {
+        let mut lock = CONTEXT_SEND.addr.lock().unwrap();
+        match lock.as_mut() {
+            Some(context) => f(context),
+            None => Ok(()),
         }
     }
 }

libs/clipboard/src/lib.rs

@@ -1,6 +1,13 @@
-use cliprdr::*;
+#[allow(dead_code)]
+use std::{
+    path::PathBuf,
+    sync::{Arc, Mutex, RwLock},
+};
+
+#[cfg(any(target_os = "windows", feature = "unix-file-copy-paste",))]
+use hbb_common::{allow_err, log};
 use hbb_common::{
-    allow_err, lazy_static, log,
+    lazy_static,
     tokio::sync::{
         mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender},
         Mutex as TokioMutex,
@@ -8,19 +15,59 @@ use hbb_common::{
     ResultType,
 };
 use serde_derive::{Deserialize, Serialize};
-use std::{
-    boxed::Box,
-    ffi::{CStr, CString},
-    sync::{Arc, Mutex, RwLock},
-};
+use thiserror::Error;
 
-pub mod cliprdr;
 pub mod context_send;
+pub mod platform;
 pub use context_send::*;
 
+#[cfg(target_os = "windows")]
 const ERR_CODE_SERVER_FUNCTION_NONE: u32 = 0x00000001;
+#[cfg(target_os = "windows")]
 const ERR_CODE_INVALID_PARAMETER: u32 = 0x00000002;
 
+pub(crate) use platform::create_cliprdr_context;
+
+/// Ability to handle Clipboard File from remote rustdesk client
+///
+/// # Note
+/// There actually should be 2 parts to implement a useable clipboard file service,
+/// but this only contains the RPC server part.
+/// The local listener and transport part is too platform specific to wrap up in typeclasses.
+pub trait CliprdrServiceContext: Send + Sync {
+    /// set to be stopped
+    fn set_is_stopped(&mut self) -> Result<(), CliprdrError>;
+    /// clear the content on clipboard
+    fn empty_clipboard(&mut self, conn_id: i32) -> Result<bool, CliprdrError>;
+
+    /// run as a server for clipboard RPC
+    fn server_clip_file(&mut self, conn_id: i32, msg: ClipboardFile) -> Result<(), CliprdrError>;
+}
+
+#[derive(Error, Debug)]
+pub enum CliprdrError {
+    #[error("invalid cliprdr name")]
+    CliprdrName,
+    #[error("failed to init cliprdr")]
+    CliprdrInit,
+    #[error("cliprdr out of memory")]
+    CliprdrOutOfMemory,
+    #[error("cliprdr internal error")]
+    ClipboardInternalError,
+    #[error("cliprdr occupied")]
+    ClipboardOccupied,
+    #[error("conversion failure")]
+    ConversionFailure,
+    #[error("failure to read clipboard")]
+    OpenClipboard,
+    #[error("failure to read file metadata or content")]
+    FileError { path: PathBuf, err: std::io::Error },
+    #[error("invalid request")]
+    InvalidRequest { description: String },
+    #[error("unknown cliprdr error")]
+    Unknown(u32),
+}
+
 #[derive(Debug, Serialize, Deserialize, Clone)]
 #[serde(tag = "t", content = "c")]
 pub enum ClipboardFile {
@@ -63,6 +110,7 @@ pub enum ClipboardFile {
 struct MsgChannel {
     peer_id: String,
     conn_id: i32,
+    #[allow(dead_code)]
     sender: UnboundedSender<ClipboardFile>,
     receiver: Arc<TokioMutex<UnboundedReceiver<ClipboardFile>>>,
 }
@@ -74,19 +122,19 @@ lazy_static::lazy_static! {
 
 impl ClipboardFile {
     pub fn is_stopping_allowed(&self) -> bool {
-        match self {
+        matches!(
+            self,
             ClipboardFile::MonitorReady
-            | ClipboardFile::FormatList { .. }
-            | ClipboardFile::FormatDataRequest { .. } => true,
-            _ => false,
-        }
+                | ClipboardFile::FormatList { .. }
+                | ClipboardFile::FormatDataRequest { .. }
+        )
     }
 
     pub fn is_stopping_allowed_from_peer(&self) -> bool {
-        match self {
-            ClipboardFile::MonitorReady | ClipboardFile::FormatList { .. } => true,
-            _ => false,
-        }
+        matches!(
+            self,
+            ClipboardFile::MonitorReady | ClipboardFile::FormatList { .. }
+        )
     }
 }
 
@@ -148,8 +196,21 @@ pub fn get_rx_cliprdr_server(conn_id: i32) -> Arc<TokioMutex<UnboundedReceiver<C
     }
 }
 
+#[cfg(any(target_os = "windows", feature = "unix-file-copy-paste",))]
 #[inline]
 fn send_data(conn_id: i32, data: ClipboardFile) {
+    #[cfg(target_os = "windows")]
+    return send_data_to_channel(conn_id, data);
+    #[cfg(not(target_os = "windows"))]
+    if conn_id == 0 {
+        send_data_to_all(data);
+    } else {
+        send_data_to_channel(conn_id, data);
+    }
+}
+#[cfg(any(target_os = "windows", feature = "unix-file-copy-paste",))]
+#[inline]
+fn send_data_to_channel(conn_id: i32, data: ClipboardFile) {
     // no need to handle result here
     if let Some(msg_channel) = VEC_MSG_CHANNEL
         .read()
@@ -161,608 +222,13 @@ fn send_data(conn_id: i32, data: ClipboardFile) {
     }
 }
 
-pub fn empty_clipboard(context: &mut CliprdrClientContext, conn_id: i32) -> bool {
-    unsafe { TRUE == cliprdr::empty_cliprdr(context, conn_id as u32) }
-}
-
-pub fn server_clip_file(
-    context: &mut CliprdrClientContext,
-    conn_id: i32,
-    msg: ClipboardFile,
-) -> u32 {
-    let mut ret = 0;
-    match msg {
-        ClipboardFile::NotifyCallback { .. } => {
-            // unreachable
-        }
-        ClipboardFile::MonitorReady => {
-            log::debug!("server_monitor_ready called");
-            ret = server_monitor_ready(context, conn_id);
-            log::debug!(
-                "server_monitor_ready called, conn_id {}, return {}",
-                conn_id,
-                ret
-            );
-        }
-        ClipboardFile::FormatList { format_list } => {
-            log::debug!(
-                "server_format_list called, conn_id {}, format_list: {:?}",
-                conn_id,
-                &format_list
-            );
-            ret = server_format_list(context, conn_id, format_list);
-            log::debug!(
-                "server_format_list called, conn_id {}, return {}",
-                conn_id,
-                ret
-            );
-        }
-        ClipboardFile::FormatListResponse { msg_flags } => {
-            log::debug!("server_format_list_response called");
-            ret = server_format_list_response(context, conn_id, msg_flags);
-            log::debug!(
-                "server_format_list_response called, conn_id {}, msg_flags {}, return {}",
-                conn_id,
-                msg_flags,
-                ret
-            );
-        }
-        ClipboardFile::FormatDataRequest {
-            requested_format_id,
-        } => {
-            log::debug!("server_format_data_request called");
-            ret = server_format_data_request(context, conn_id, requested_format_id);
-            log::debug!(
-                "server_format_data_request called, conn_id {}, requested_format_id {}, return {}",
-                conn_id,
-                requested_format_id,
-                ret
-            );
-        }
-        ClipboardFile::FormatDataResponse {
-            msg_flags,
-            format_data,
-        } => {
-            log::debug!("server_format_data_response called");
-            ret = server_format_data_response(context, conn_id, msg_flags, format_data);
-            log::debug!(
-                "server_format_data_response called, conn_id {}, msg_flags: {}, return {}",
-                conn_id,
-                msg_flags,
-                ret
-            );
-        }
-        ClipboardFile::FileContentsRequest {
-            stream_id,
-            list_index,
-            dw_flags,
-            n_position_low,
-            n_position_high,
-            cb_requested,
-            have_clip_data_id,
-            clip_data_id,
-        } => {
-            log::debug!("server_file_contents_request called");
-            ret = server_file_contents_request(
-                context,
-                conn_id,
-                stream_id,
-                list_index,
-                dw_flags,
-                n_position_low,
-                n_position_high,
-                cb_requested,
-                have_clip_data_id,
-                clip_data_id,
-            );
-            log::debug!("server_file_contents_request called, conn_id {}, stream_id: {}, list_index {}, dw_flags {}, n_position_low {}, n_position_high {}, cb_requested {}, have_clip_data_id {}, clip_data_id {}, return {}",                 conn_id,
-                stream_id,
-                list_index,
-                dw_flags,
-                n_position_low,
-                n_position_high,
-                cb_requested,
-                have_clip_data_id,
-                clip_data_id,
-                ret
-            );
-        }
-        ClipboardFile::FileContentsResponse {
-            msg_flags,
-            stream_id,
-            requested_data,
-        } => {
-            log::debug!("server_file_contents_response called");
-            ret = server_file_contents_response(
-                context,
-                conn_id,
-                msg_flags,
-                stream_id,
-                requested_data,
-            );
-            log::debug!("server_file_contents_response called, conn_id {}, msg_flags {}, stream_id {}, return {}",
-                conn_id,
-                msg_flags,
-                stream_id,
-                ret
-            );
-        }
-    }
-    ret
-}
-
-pub fn server_monitor_ready(context: &mut CliprdrClientContext, conn_id: i32) -> u32 {
-    unsafe {
-        let monitor_ready = CLIPRDR_MONITOR_READY {
-            connID: conn_id as UINT32,
-            msgType: 0 as UINT16,
-            msgFlags: 0 as UINT16,
-            dataLen: 0 as UINT32,
-        };
-        if let Some(f) = context.MonitorReady {
-            let ret = f(context, &monitor_ready);
-            ret as u32
-        } else {
-            ERR_CODE_SERVER_FUNCTION_NONE
-        }
-    }
-}
-
-pub fn server_format_list(
-    context: &mut CliprdrClientContext,
-    conn_id: i32,
-    format_list: Vec<(i32, String)>,
-) -> u32 {
-    unsafe {
-        let num_formats = format_list.len() as UINT32;
-        let mut formats = format_list
-            .into_iter()
-            .map(|format| {
-                if format.1.is_empty() {
-                    CLIPRDR_FORMAT {
-                        formatId: format.0 as UINT32,
-                        formatName: 0 as *mut _,
-                    }
-                } else {
-                    let n = match CString::new(format.1) {
-                        Ok(n) => n,
-                        Err(_) => CString::new("").unwrap(),
-                    };
-                    CLIPRDR_FORMAT {
-                        formatId: format.0 as UINT32,
-                        formatName: n.into_raw(),
-                    }
-                }
-            })
-            .collect::<Vec<CLIPRDR_FORMAT>>();
-
-        let format_list = CLIPRDR_FORMAT_LIST {
-            connID: conn_id as UINT32,
-            msgType: 0 as UINT16,
-            msgFlags: 0 as UINT16,
-            dataLen: 0 as UINT32,
-            numFormats: num_formats,
-            formats: formats.as_mut_ptr(),
-        };
-
-        let ret = if let Some(f) = context.ServerFormatList {
-            f(context, &format_list)
-        } else {
-            ERR_CODE_SERVER_FUNCTION_NONE
-        };
-
-        for f in formats {
-            if !f.formatName.is_null() {
-                // retake pointer to free memory
-                let _ = CString::from_raw(f.formatName);
-            }
-        }
-
-        ret as u32
-    }
-}
-
-pub fn server_format_list_response(
-    context: &mut CliprdrClientContext,
-    conn_id: i32,
-    msg_flags: i32,
-) -> u32 {
-    unsafe {
-        let format_list_response = CLIPRDR_FORMAT_LIST_RESPONSE {
-            connID: conn_id as UINT32,
-            msgType: 0 as UINT16,
-            msgFlags: msg_flags as UINT16,
-            dataLen: 0 as UINT32,
-        };
-
-        if let Some(f) = context.ServerFormatListResponse {
-            f(context, &format_list_response)
-        } else {
-            ERR_CODE_SERVER_FUNCTION_NONE
-        }
-    }
-}
-
-pub fn server_format_data_request(
-    context: &mut CliprdrClientContext,
-    conn_id: i32,
-    requested_format_id: i32,
-) -> u32 {
-    unsafe {
-        let format_data_request = CLIPRDR_FORMAT_DATA_REQUEST {
-            connID: conn_id as UINT32,
-            msgType: 0 as UINT16,
-            msgFlags: 0 as UINT16,
-            dataLen: 0 as UINT32,
-            requestedFormatId: requested_format_id as UINT32,
-        };
-        if let Some(f) = context.ServerFormatDataRequest {
-            f(context, &format_data_request)
-        } else {
-            ERR_CODE_SERVER_FUNCTION_NONE
-        }
-    }
-}
-
-pub fn server_format_data_response(
-    context: &mut CliprdrClientContext,
-    conn_id: i32,
-    msg_flags: i32,
-    mut format_data: Vec<u8>,
-) -> u32 {
-    unsafe {
-        let format_data_response = CLIPRDR_FORMAT_DATA_RESPONSE {
-            connID: conn_id as UINT32,
-            msgType: 0 as UINT16,
-            msgFlags: msg_flags as UINT16,
-            dataLen: format_data.len() as UINT32,
-            requestedFormatData: format_data.as_mut_ptr(),
-        };
-        if let Some(f) = context.ServerFormatDataResponse {
-            f(context, &format_data_response)
-        } else {
-            ERR_CODE_SERVER_FUNCTION_NONE
-        }
-    }
-}
-
-pub fn server_file_contents_request(
-    context: &mut CliprdrClientContext,
-    conn_id: i32,
-    stream_id: i32,
-    list_index: i32,
-    dw_flags: i32,
-    n_position_low: i32,
-    n_position_high: i32,
-    cb_requested: i32,
-    have_clip_data_id: bool,
-    clip_data_id: i32,
-) -> u32 {
-    unsafe {
-        let file_contents_request = CLIPRDR_FILE_CONTENTS_REQUEST {
-            connID: conn_id as UINT32,
-            msgType: 0 as UINT16,
-            msgFlags: 0 as UINT16,
-            dataLen: 0 as UINT32,
-            streamId: stream_id as UINT32,
-            listIndex: list_index as UINT32,
-            dwFlags: dw_flags as UINT32,
-            nPositionLow: n_position_low as UINT32,
-            nPositionHigh: n_position_high as UINT32,
-            cbRequested: cb_requested as UINT32,
-            haveClipDataId: if have_clip_data_id { TRUE } else { FALSE },
-            clipDataId: clip_data_id as UINT32,
-        };
-        if let Some(f) = context.ServerFileContentsRequest {
-            f(context, &file_contents_request)
-        } else {
-            ERR_CODE_SERVER_FUNCTION_NONE
-        }
-    }
-}
-
-pub fn server_file_contents_response(
-    context: &mut CliprdrClientContext,
-    conn_id: i32,
-    msg_flags: i32,
-    stream_id: i32,
-    mut requested_data: Vec<u8>,
-) -> u32 {
-    unsafe {
-        let file_contents_response = CLIPRDR_FILE_CONTENTS_RESPONSE {
-            connID: conn_id as UINT32,
-            msgType: 0 as UINT16,
-            msgFlags: msg_flags as UINT16,
-            dataLen: 4 + requested_data.len() as UINT32,
-            streamId: stream_id as UINT32,
-            cbRequested: requested_data.len() as UINT32,
-            requestedData: requested_data.as_mut_ptr(),
-        };
-        if let Some(f) = context.ServerFileContentsResponse {
-            f(context, &file_contents_response)
-        } else {
-            ERR_CODE_SERVER_FUNCTION_NONE
-        }
-    }
-}
-
-pub fn create_cliprdr_context(
-    enable_files: bool,
-    enable_others: bool,
-    response_wait_timeout_secs: u32,
-) -> ResultType<Box<CliprdrClientContext>> {
-    Ok(CliprdrClientContext::create(
-        enable_files,
-        enable_others,
-        response_wait_timeout_secs,
-        Some(notify_callback),
-        Some(client_format_list),
-        Some(client_format_list_response),
-        Some(client_format_data_request),
-        Some(client_format_data_response),
-        Some(client_file_contents_request),
-        Some(client_file_contents_response),
-    )?)
-}
-
-extern "C" fn notify_callback(conn_id: UINT32, msg: *const NOTIFICATION_MESSAGE) -> UINT {
-    log::debug!("notify_callback called");
-    let data = unsafe {
-        let msg = &*msg;
-        let details = if msg.details.is_null() {
-            Ok("")
-        } else {
-            CStr::from_ptr(msg.details as _).to_str()
-        };
-        match (CStr::from_ptr(msg.msg as _).to_str(), details) {
-            (Ok(m), Ok(d)) => {
-                let msgtype = format!(
-                    "custom-{}-nocancel-nook-hasclose",
-                    if msg.r#type == 0 {
-                        "info"
-                    } else if msg.r#type == 1 {
-                        "warn"
-                    } else {
-                        "error"
-                    }
-                );
-                let title = "Clipboard";
-                let text = if d.is_empty() {
-                    m.to_string()
-                } else {
-                    format!("{} {}", m, d)
-                };
-                ClipboardFile::NotifyCallback {
-                    r#type: msgtype,
-                    title: title.to_string(),
-                    text,
-                }
-            }
-            _ => {
-                log::error!("notify_callback: failed to convert msg");
-                return ERR_CODE_INVALID_PARAMETER;
-            }
-        }
-    };
+#[cfg(feature = "unix-file-copy-paste")]
+#[inline]
+fn send_data_to_all(data: ClipboardFile) {
     // no need to handle result here
-    send_data(conn_id as _, data);
-
-    0
-}
-
-extern "C" fn client_format_list(
-    _context: *mut CliprdrClientContext,
-    clip_format_list: *const CLIPRDR_FORMAT_LIST,
-) -> UINT {
-    let conn_id;
-    let mut format_list: Vec<(i32, String)> = Vec::new();
-    unsafe {
-        let mut i = 0u32;
-        while i < (*clip_format_list).numFormats {
-            let format_data = &(*(*clip_format_list).formats.offset(i as isize));
-            if format_data.formatName.is_null() {
-                format_list.push((format_data.formatId as i32, "".to_owned()));
-            } else {
-                let format_name = CStr::from_ptr(format_data.formatName).to_str();
-                let format_name = match format_name {
-                    Ok(n) => n.to_owned(),
-                    Err(_) => {
-                        log::warn!("failed to get format name");
-                        "".to_owned()
-                    }
-                };
-                format_list.push((format_data.formatId as i32, format_name));
-            }
-            // log::debug!("format list item {}: format id: {}, format name: {}", i, format_data.formatId, &format_name);
-            i += 1;
-        }
-        conn_id = (*clip_format_list).connID as i32;
+    for msg_channel in VEC_MSG_CHANNEL.read().unwrap().iter() {
+        allow_err!(msg_channel.sender.send(data.clone()));
     }
-    log::debug!(
-        "client_format_list called, client id: {}, format_list: {:?}",
-        conn_id,
-        &format_list
-    );
-    let data = ClipboardFile::FormatList { format_list };
-    // no need to handle result here
-    if conn_id == 0 {
-        // msg_channel is used for debug, VEC_MSG_CHANNEL cannot be inspected by the debugger.
-        let msg_channel = VEC_MSG_CHANNEL.read().unwrap();
-        msg_channel
-            .iter()
-            .for_each(|msg_channel| allow_err!(msg_channel.sender.send(data.clone())));
-    } else {
-        send_data(conn_id, data);
-    }
-
-    0
-}
-
-extern "C" fn client_format_list_response(
-    _context: *mut CliprdrClientContext,
-    format_list_response: *const CLIPRDR_FORMAT_LIST_RESPONSE,
-) -> UINT {
-    let conn_id;
-    let msg_flags;
-    unsafe {
-        conn_id = (*format_list_response).connID as i32;
-        msg_flags = (*format_list_response).msgFlags as i32;
-    }
-    log::debug!(
-        "client_format_list_response called, client id: {}, msg_flags: {}",
-        conn_id,
-        msg_flags
-    );
-    let data = ClipboardFile::FormatListResponse { msg_flags };
-    send_data(conn_id, data);
-
-    0
-}
-
-extern "C" fn client_format_data_request(
-    _context: *mut CliprdrClientContext,
-    format_data_request: *const CLIPRDR_FORMAT_DATA_REQUEST,
-) -> UINT {
-    let conn_id;
-    let requested_format_id;
-    unsafe {
-        conn_id = (*format_data_request).connID as i32;
-        requested_format_id = (*format_data_request).requestedFormatId as i32;
-    }
-    let data = ClipboardFile::FormatDataRequest {
-        requested_format_id,
-    };
-    log::debug!(
-        "client_format_data_request called, conn_id: {}, requested_format_id: {}",
-        conn_id,
-        requested_format_id
-    );
-    // no need to handle result here
-    send_data(conn_id, data);
-
-    0
-}
-
-extern "C" fn client_format_data_response(
-    _context: *mut CliprdrClientContext,
-    format_data_response: *const CLIPRDR_FORMAT_DATA_RESPONSE,
-) -> UINT {
-    let conn_id;
-    let msg_flags;
-    let format_data;
-    unsafe {
-        conn_id = (*format_data_response).connID as i32;
-        msg_flags = (*format_data_response).msgFlags as i32;
-        if (*format_data_response).requestedFormatData.is_null() {
-            format_data = Vec::new();
-        } else {
-            format_data = std::slice::from_raw_parts(
-                (*format_data_response).requestedFormatData,
-                (*format_data_response).dataLen as usize,
-            )
-            .to_vec();
-        }
-    }
-    log::debug!(
-        "client_format_data_response called, client id: {}, msg_flags: {}",
-        conn_id,
-        msg_flags
-    );
-    let data = ClipboardFile::FormatDataResponse {
-        msg_flags,
-        format_data,
-    };
-    send_data(conn_id, data);
-
-    0
-}
-
-extern "C" fn client_file_contents_request(
-    _context: *mut CliprdrClientContext,
-    file_contents_request: *const CLIPRDR_FILE_CONTENTS_REQUEST,
-) -> UINT {
-    // TODO: support huge file?
-    // if (!cliprdr->hasHugeFileSupport)
-    // {
-    // 	if (((UINT64)fileContentsRequest->cbRequested + fileContentsRequest->nPositionLow) >
-    // 	    UINT32_MAX)
-    // 		return ERROR_INVALID_PARAMETER;
-    // 	if (fileContentsRequest->nPositionHigh != 0)
-    // 		return ERROR_INVALID_PARAMETER;
-    // }
-
-    let conn_id;
-    let stream_id;
-    let list_index;
-    let dw_flags;
-    let n_position_low;
-    let n_position_high;
-    let cb_requested;
-    let have_clip_data_id;
-    let clip_data_id;
-    unsafe {
-        conn_id = (*file_contents_request).connID as i32;
-        stream_id = (*file_contents_request).streamId as i32;
-        list_index = (*file_contents_request).listIndex as i32;
-        dw_flags = (*file_contents_request).dwFlags as i32;
-        n_position_low = (*file_contents_request).nPositionLow as i32;
-        n_position_high = (*file_contents_request).nPositionHigh as i32;
-        cb_requested = (*file_contents_request).cbRequested as i32;
-        have_clip_data_id = (*file_contents_request).haveClipDataId == TRUE;
-        clip_data_id = (*file_contents_request).clipDataId as i32;
-    }
-    let data = ClipboardFile::FileContentsRequest {
-        stream_id,
-        list_index,
-        dw_flags,
-        n_position_low,
-        n_position_high,
-        cb_requested,
-        have_clip_data_id,
-        clip_data_id,
-    };
-    log::debug!("client_file_contents_request called, data: {:?}", &data);
-    send_data(conn_id, data);
-
-    0
-}
-
-extern "C" fn client_file_contents_response(
-    _context: *mut CliprdrClientContext,
-    file_contents_response: *const CLIPRDR_FILE_CONTENTS_RESPONSE,
-) -> UINT {
-    let conn_id;
-    let msg_flags;
-    let stream_id;
-    let requested_data;
-    unsafe {
-        conn_id = (*file_contents_response).connID as i32;
-        msg_flags = (*file_contents_response).msgFlags as i32;
-        stream_id = (*file_contents_response).streamId as i32;
-        if (*file_contents_response).requestedData.is_null() {
-            requested_data = Vec::new();
-        } else {
-            requested_data = std::slice::from_raw_parts(
-                (*file_contents_response).requestedData,
-                (*file_contents_response).cbRequested as usize,
-            )
-            .to_vec();
-        }
-    }
-    let data = ClipboardFile::FileContentsResponse {
-        msg_flags,
-        stream_id,
-        requested_data,
-    };
-    log::debug!(
-        "client_file_contents_response called, conn_id: {}, msg_flags: {}, stream_id: {}",
-        conn_id,
-        msg_flags,
-        stream_id
-    );
-    send_data(conn_id, data);
-
-    0
 }
 
 #[cfg(test)]

libs/clipboard/src/platform/mod.rs

@@ -0,0 +1,88 @@
+use crate::{CliprdrError, CliprdrServiceContext};
+
+#[cfg(target_os = "windows")]
+pub mod windows;
+#[cfg(target_os = "windows")]
+pub fn create_cliprdr_context(
+    enable_files: bool,
+    enable_others: bool,
+    response_wait_timeout_secs: u32,
+) -> crate::ResultType<Box<dyn crate::CliprdrServiceContext>> {
+    let boxed =
+        windows::create_cliprdr_context(enable_files, enable_others, response_wait_timeout_secs)?
+            as Box<_>;
+    Ok(boxed)
+}
+
+#[cfg(feature = "unix-file-copy-paste")]
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+/// use FUSE for file pasting on these platforms
+pub mod fuse;
+#[cfg(feature = "unix-file-copy-paste")]
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+pub mod unix;
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+pub fn create_cliprdr_context(
+    _enable_files: bool,
+    _enable_others: bool,
+    _response_wait_timeout_secs: u32,
+) -> crate::ResultType<Box<dyn crate::CliprdrServiceContext>> {
+    #[cfg(feature = "unix-file-copy-paste")]
+    {
+        use std::{fs::Permissions, os::unix::prelude::PermissionsExt};
+
+        use hbb_common::{config::APP_NAME, log};
+
+        if !_enable_files {
+            return Ok(Box::new(DummyCliprdrContext {}) as Box<_>);
+        }
+
+        let timeout = std::time::Duration::from_secs(_response_wait_timeout_secs as u64);
+
+        let app_name = APP_NAME.read().unwrap().clone();
+
+        let mnt_path = format!("/tmp/{}/{}", app_name, "cliprdr");
+
+        // this function must be called after the main IPC is up
+        std::fs::create_dir(&mnt_path).ok();
+        std::fs::set_permissions(&mnt_path, Permissions::from_mode(0o777)).ok();
+
+        log::info!("clear previously mounted cliprdr FUSE");
+        if let Err(e) = std::process::Command::new("umount").arg(&mnt_path).status() {
+            log::warn!("umount {:?} may fail: {:?}", mnt_path, e);
+        }
+
+        let unix_ctx = unix::ClipboardContext::new(timeout, mnt_path.parse().unwrap())?;
+        log::debug!("start cliprdr FUSE");
+        unix_ctx.run().expect("failed to start cliprdr FUSE");
+
+        Ok(Box::new(unix_ctx) as Box<_>)
+    }
+
+    #[cfg(not(feature = "unix-file-copy-paste"))]
+    return Ok(Box::new(DummyCliprdrContext {}) as Box<_>);
+}
+
+struct DummyCliprdrContext {}
+
+impl CliprdrServiceContext for DummyCliprdrContext {
+    fn set_is_stopped(&mut self) -> Result<(), CliprdrError> {
+        Ok(())
+    }
+    fn empty_clipboard(&mut self, _conn_id: i32) -> Result<bool, CliprdrError> {
+        Ok(true)
+    }
+    fn server_clip_file(
+        &mut self,
+        _conn_id: i32,
+        _msg: crate::ClipboardFile,
+    ) -> Result<(), crate::CliprdrError> {
+        Ok(())
+    }
+}
+
+#[cfg(feature = "unix-file-copy-paste")]
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+// begin of epoch used by microsoft
+// 1601-01-01 00:00:00 + LDAP_EPOCH_DELTA*(100 ns) = 1970-01-01 00:00:00
+const LDAP_EPOCH_DELTA: u64 = 116444772610000000;

libs/clipboard/src/platform/unix/local_file.rs

@@ -0,0 +1,367 @@
+use std::{
+    collections::HashSet,
+    fs::File,
+    io::{BufRead, BufReader, Read, Seek},
+    os::unix::prelude::PermissionsExt,
+    path::PathBuf,
+    sync::atomic::{AtomicU64, Ordering},
+    time::SystemTime,
+};
+
+use hbb_common::{
+    bytes::{BufMut, BytesMut},
+    log,
+};
+use utf16string::WString;
+
+use crate::{
+    platform::{fuse::BLOCK_SIZE, LDAP_EPOCH_DELTA},
+    CliprdrError,
+};
+
+/// has valid file attributes
+const FLAGS_FD_ATTRIBUTES: u32 = 0x04;
+/// has valid file size
+const FLAGS_FD_SIZE: u32 = 0x40;
+/// has valid last write time
+const FLAGS_FD_LAST_WRITE: u32 = 0x20;
+/// show progress
+const FLAGS_FD_PROGRESSUI: u32 = 0x4000;
+/// transferred from unix, contains file mode
+/// P.S. this flag is not used in windows
+const FLAGS_FD_UNIX_MODE: u32 = 0x08;
+
+#[derive(Debug)]
+pub(super) struct LocalFile {
+    pub path: PathBuf,
+
+    pub handle: Option<BufReader<File>>,
+    pub offset: AtomicU64,
+
+    pub name: String,
+    pub size: u64,
+    pub last_write_time: SystemTime,
+    pub is_dir: bool,
+    pub perm: u32,
+    pub read_only: bool,
+    pub hidden: bool,
+    pub system: bool,
+    pub archive: bool,
+    pub normal: bool,
+}
+
+impl LocalFile {
+    pub fn try_open(path: &PathBuf) -> Result<Self, CliprdrError> {
+        let mt = std::fs::metadata(path).map_err(|e| CliprdrError::FileError {
+            path: path.clone(),
+            err: e,
+        })?;
+        let size = mt.len() as u64;
+        let is_dir = mt.is_dir();
+        let read_only = mt.permissions().readonly();
+        let system = false;
+        let hidden = path.to_string_lossy().starts_with('.');
+        let archive = false;
+        let normal = !(is_dir || read_only || system || hidden || archive);
+        let last_write_time = mt.modified().unwrap_or(SystemTime::UNIX_EPOCH);
+
+        let perm = mt.permissions().mode();
+
+        let name = path
+            .display()
+            .to_string()
+            .trim_start_matches('/')
+            .replace('/', "\\");
+
+        // NOTE: open files lazily
+        let handle = None;
+        let offset = AtomicU64::new(0);
+
+        Ok(Self {
+            name,
+            path: path.clone(),
+            handle,
+            offset,
+            size,
+            last_write_time,
+            is_dir,
+            read_only,
+            system,
+            hidden,
+            perm,
+            archive,
+            normal,
+        })
+    }
+    pub fn as_bin(&self) -> Vec<u8> {
+        let mut buf = BytesMut::with_capacity(592);
+
+        let read_only_flag = if self.read_only { 0x1 } else { 0 };
+        let hidden_flag = if self.hidden { 0x2 } else { 0 };
+        let system_flag = if self.system { 0x4 } else { 0 };
+        let directory_flag = if self.is_dir { 0x10 } else { 0 };
+        let archive_flag = if self.archive { 0x20 } else { 0 };
+        let normal_flag = if self.normal { 0x80 } else { 0 };
+
+        let file_attributes: u32 = read_only_flag
+            | hidden_flag
+            | system_flag
+            | directory_flag
+            | archive_flag
+            | normal_flag;
+
+        let win32_time = self
+            .last_write_time
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_nanos() as u64
+            / 100
+            + LDAP_EPOCH_DELTA;
+
+        let size_high = (self.size >> 32) as u32;
+        let size_low = (self.size & (u32::MAX as u64)) as u32;
+
+        let path = self.path.to_string_lossy().to_string();
+
+        let wstr: WString<utf16string::LE> = WString::from(&path);
+        let name = wstr.as_bytes();
+
+        log::trace!(
+            "put file to list: name_len {}, name {}",
+            name.len(),
+            &self.name
+        );
+
+        let flags = FLAGS_FD_SIZE
+            | FLAGS_FD_LAST_WRITE
+            | FLAGS_FD_ATTRIBUTES
+            | FLAGS_FD_PROGRESSUI
+            | FLAGS_FD_UNIX_MODE;
+
+        // flags, 4 bytes
+        buf.put_u32_le(flags);
+        // 32 bytes reserved
+        buf.put(&[0u8; 32][..]);
+        // file attributes, 4 bytes
+        buf.put_u32_le(file_attributes);
+
+        // NOTE: this is not used in windows
+        // in the specification, this is 16 bytes reserved
+        // lets use the last 4 bytes to store the file mode
+        //
+        // 12 bytes reserved
+        buf.put(&[0u8; 12][..]);
+        // file permissions, 4 bytes
+        buf.put_u32_le(self.perm);
+
+        // last write time, 8 bytes
+        buf.put_u64_le(win32_time);
+        // file size (high)
+        buf.put_u32_le(size_high);
+        // file size (low)
+        buf.put_u32_le(size_low);
+        // put name and padding to 520 bytes
+        let name_len = name.len();
+        buf.put(name);
+        buf.put(&vec![0u8; 520 - name_len][..]);
+
+        buf.to_vec()
+    }
+
+    #[inline]
+    pub fn load_handle(&mut self) -> Result<(), CliprdrError> {
+        if !self.is_dir && self.handle.is_none() {
+            let handle = std::fs::File::open(&self.path).map_err(|e| CliprdrError::FileError {
+                path: self.path.clone(),
+                err: e,
+            })?;
+            let mut reader = BufReader::with_capacity(BLOCK_SIZE as usize * 2, handle);
+            reader.fill_buf().map_err(|e| CliprdrError::FileError {
+                path: self.path.clone(),
+                err: e,
+            })?;
+            self.handle = Some(reader);
+        };
+        Ok(())
+    }
+
+    pub fn read_exact_at(&mut self, buf: &mut [u8], offset: u64) -> Result<(), CliprdrError> {
+        self.load_handle()?;
+
+        let handle = self.handle.as_mut().unwrap();
+
+        if offset != self.offset.load(Ordering::Relaxed) {
+            handle
+                .seek(std::io::SeekFrom::Start(offset))
+                .map_err(|e| CliprdrError::FileError {
+                    path: self.path.clone(),
+                    err: e,
+                })?;
+        }
+        handle
+            .read_exact(buf)
+            .map_err(|e| CliprdrError::FileError {
+                path: self.path.clone(),
+                err: e,
+            })?;
+        let new_offset = offset + (buf.len() as u64);
+        self.offset.store(new_offset, Ordering::Relaxed);
+
+        // gc file handle
+        if new_offset >= self.size {
+            self.offset.store(0, Ordering::Relaxed);
+            self.handle = None;
+        }
+
+        Ok(())
+    }
+}
+
+pub(super) fn construct_file_list(paths: &[PathBuf]) -> Result<Vec<LocalFile>, CliprdrError> {
+    fn constr_file_lst(
+        path: &PathBuf,
+        file_list: &mut Vec<LocalFile>,
+        visited: &mut HashSet<PathBuf>,
+    ) -> Result<(), CliprdrError> {
+        // prevent fs loop
+        if visited.contains(path) {
+            return Ok(());
+        }
+        visited.insert(path.clone());
+
+        let local_file = LocalFile::try_open(path)?;
+        file_list.push(local_file);
+
+        let mt = std::fs::metadata(path).map_err(|e| CliprdrError::FileError {
+            path: path.clone(),
+            err: e,
+        })?;
+
+        if mt.is_dir() {
+            let dir = std::fs::read_dir(path).unwrap();
+            for entry in dir {
+                let entry = entry.unwrap();
+                let path = entry.path();
+                constr_file_lst(&path, file_list, visited)?;
+            }
+        }
+        Ok(())
+    }
+
+    let mut file_list = Vec::new();
+    let mut visited = HashSet::new();
+
+    for path in paths {
+        constr_file_lst(path, &mut file_list, &mut visited)?;
+    }
+    Ok(file_list)
+}
+
+#[cfg(test)]
+mod file_list_test {
+    use std::{path::PathBuf, sync::atomic::AtomicU64};
+
+    use hbb_common::bytes::{BufMut, BytesMut};
+
+    use crate::{platform::fuse::FileDescription, CliprdrError};
+
+    use super::LocalFile;
+
+    #[inline]
+    fn generate_tree(prefix: &str) -> Vec<LocalFile> {
+        // generate a tree of local files, no handles
+        // - /
+        // |- a.txt
+        // |- b
+        //    |- c.txt
+        #[inline]
+        fn generate_file(path: &str, name: &str, is_dir: bool) -> LocalFile {
+            LocalFile {
+                path: PathBuf::from(path),
+                handle: None,
+                name: name.to_string(),
+                size: 0,
+                offset: AtomicU64::new(0),
+                last_write_time: std::time::SystemTime::UNIX_EPOCH,
+                read_only: false,
+                is_dir,
+                perm: 0o754,
+                hidden: false,
+                system: false,
+                archive: false,
+                normal: false,
+            }
+        }
+
+        let p = prefix;
+
+        let (r_path, a_path, b_path, c_path) = if !prefix.is_empty() {
+            (
+                p.to_string(),
+                format!("{}/a.txt", p),
+                format!("{}/b", p),
+                format!("{}/b/c.txt", p),
+            )
+        } else {
+            (
+                ".".to_string(),
+                "a.txt".to_string(),
+                "b".to_string(),
+                "b/c.txt".to_string(),
+            )
+        };
+
+        let root = generate_file(&r_path, ".", true);
+        let a = generate_file(&a_path, "a.txt", false);
+        let b = generate_file(&b_path, "b", true);
+        let c = generate_file(&c_path, "c.txt", false);
+
+        vec![root, a, b, c]
+    }
+
+    fn as_bin_parse_test(prefix: &str) -> Result<(), CliprdrError> {
+        let tree = generate_tree(prefix);
+        let mut pdu = BytesMut::with_capacity(4 + 592 * tree.len());
+        pdu.put_u32_le(tree.len() as u32);
+        for file in tree {
+            pdu.put(file.as_bin().as_slice());
+        }
+
+        let parsed = FileDescription::parse_file_descriptors(pdu.to_vec(), 0)?;
+        assert_eq!(parsed.len(), 4);
+
+        if !prefix.is_empty() {
+            assert_eq!(parsed[0].name.to_str().unwrap(), format!("{}", prefix));
+            assert_eq!(
+                parsed[1].name.to_str().unwrap(),
+                format!("{}/a.txt", prefix)
+            );
+            assert_eq!(parsed[2].name.to_str().unwrap(), format!("{}/b", prefix));
+            assert_eq!(
+                parsed[3].name.to_str().unwrap(),
+                format!("{}/b/c.txt", prefix)
+            );
+        } else {
+            assert_eq!(parsed[0].name.to_str().unwrap(), ".");
+            assert_eq!(parsed[1].name.to_str().unwrap(), "a.txt");
+            assert_eq!(parsed[2].name.to_str().unwrap(), "b");
+            assert_eq!(parsed[3].name.to_str().unwrap(), "b/c.txt");
+        }
+
+        assert!(parsed[0].perm & 0o777 == 0o754);
+        assert!(parsed[1].perm & 0o777 == 0o754);
+        assert!(parsed[2].perm & 0o777 == 0o754);
+        assert!(parsed[3].perm & 0o777 == 0o754);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_file_descriptors() -> Result<(), CliprdrError> {
+        as_bin_parse_test("")?;
+        as_bin_parse_test("/")?;
+        as_bin_parse_test("test")?;
+        as_bin_parse_test("/test")?;
+        Ok(())
+    }
+}

libs/clipboard/src/platform/unix/mod.rs

@@ -0,0 +1,600 @@
+use std::{
+    path::PathBuf,
+    sync::{mpsc::Sender, Arc},
+    time::Duration,
+};
+
+use dashmap::DashMap;
+use fuser::MountOption;
+use hbb_common::{
+    bytes::{BufMut, BytesMut},
+    log,
+};
+use lazy_static::lazy_static;
+use parking_lot::Mutex;
+
+use crate::{
+    platform::{fuse::FileDescription, unix::local_file::construct_file_list},
+    send_data, ClipboardFile, CliprdrError, CliprdrServiceContext,
+};
+
+use self::local_file::LocalFile;
+#[cfg(target_os = "linux")]
+use self::url::{encode_path_to_uri, parse_plain_uri_list};
+
+use super::fuse::FuseServer;
+
+#[cfg(target_os = "linux")]
+/// clipboard implementation of x11
+pub mod x11;
+
+#[cfg(target_os = "macos")]
+/// clipboard implementation of macos
+pub mod ns_clipboard;
+
+pub mod local_file;
+
+#[cfg(target_os = "linux")]
+pub mod url;
+
+// not actual format id, just a placeholder
+const FILEDESCRIPTOR_FORMAT_ID: i32 = 49334;
+const FILEDESCRIPTORW_FORMAT_NAME: &str = "FileGroupDescriptorW";
+// not actual format id, just a placeholder
+const FILECONTENTS_FORMAT_ID: i32 = 49267;
+const FILECONTENTS_FORMAT_NAME: &str = "FileContents";
+
+lazy_static! {
+    static ref REMOTE_FORMAT_MAP: DashMap<i32, String> = DashMap::from_iter(
+        [
+            (
+                FILEDESCRIPTOR_FORMAT_ID,
+                FILEDESCRIPTORW_FORMAT_NAME.to_string()
+            ),
+            (FILECONTENTS_FORMAT_ID, FILECONTENTS_FORMAT_NAME.to_string())
+        ]
+        .iter()
+        .cloned()
+    );
+}
+
+fn get_local_format(remote_id: i32) -> Option<String> {
+    REMOTE_FORMAT_MAP.get(&remote_id).map(|s| s.clone())
+}
+
+fn add_remote_format(local_name: &str, remote_id: i32) {
+    REMOTE_FORMAT_MAP.insert(remote_id, local_name.to_string());
+}
+
+trait SysClipboard: Send + Sync {
+    fn start(&self);
+
+    fn set_file_list(&self, paths: &[PathBuf]) -> Result<(), CliprdrError>;
+    fn get_file_list(&self) -> Vec<PathBuf>;
+}
+
+#[cfg(target_os = "linux")]
+fn get_sys_clipboard(ignore_path: &PathBuf) -> Result<Box<dyn SysClipboard>, CliprdrError> {
+    #[cfg(feature = "wayland")]
+    {
+        unimplemented!()
+    }
+    #[cfg(not(feature = "wayland"))]
+    {
+        use x11::*;
+        let x11_clip = X11Clipboard::new(ignore_path)?;
+        Ok(Box::new(x11_clip) as Box<_>)
+    }
+}
+
+#[cfg(target_os = "macos")]
+fn get_sys_clipboard(ignore_path: &PathBuf) -> Result<Box<dyn SysClipboard>, CliprdrError> {
+    use ns_clipboard::*;
+    let ns_pb = NsPasteboard::new(ignore_path)?;
+    Ok(Box::new(ns_pb) as Box<_>)
+}
+
+#[derive(Debug)]
+enum FileContentsRequest {
+    Size {
+        stream_id: i32,
+        file_idx: usize,
+    },
+
+    Range {
+        stream_id: i32,
+        file_idx: usize,
+        offset: u64,
+        length: u64,
+    },
+}
+
+pub struct ClipboardContext {
+    pub fuse_mount_point: PathBuf,
+    /// stores fuse background session handle
+    fuse_handle: Mutex<Option<fuser::BackgroundSession>>,
+
+    /// a sender of clipboard file contents pdu to fuse server
+    fuse_tx: Sender<ClipboardFile>,
+    fuse_server: Arc<Mutex<FuseServer>>,
+
+    clipboard: Arc<dyn SysClipboard>,
+    local_files: Mutex<Vec<LocalFile>>,
+}
+
+impl ClipboardContext {
+    pub fn new(timeout: Duration, mount_path: PathBuf) -> Result<Self, CliprdrError> {
+        // assert mount path exists
+        let fuse_mount_point = mount_path.canonicalize().map_err(|e| {
+            log::error!("failed to canonicalize mount path: {:?}", e);
+            CliprdrError::CliprdrInit
+        })?;
+
+        let (fuse_server, fuse_tx) = FuseServer::new(timeout);
+
+        let fuse_server = Arc::new(Mutex::new(fuse_server));
+
+        let clipboard = get_sys_clipboard(&fuse_mount_point)?;
+        let clipboard = Arc::from(clipboard) as Arc<_>;
+        let local_files = Mutex::new(vec![]);
+
+        Ok(Self {
+            fuse_mount_point,
+            fuse_server,
+            fuse_tx,
+            fuse_handle: Mutex::new(None),
+            clipboard,
+            local_files,
+        })
+    }
+
+    pub fn run(&self) -> Result<(), CliprdrError> {
+        if !self.is_stopped() {
+            return Ok(());
+        }
+
+        let mut fuse_handle = self.fuse_handle.lock();
+
+        let mount_path = &self.fuse_mount_point;
+
+        let mnt_opts = [
+            MountOption::FSName("rustdesk-cliprdr-fs".to_string()),
+            MountOption::NoAtime,
+            MountOption::RO,
+        ];
+        log::info!(
+            "mounting clipboard FUSE to {}",
+            self.fuse_mount_point.display()
+        );
+
+        let new_handle = fuser::spawn_mount2(
+            FuseServer::client(self.fuse_server.clone()),
+            mount_path,
+            &mnt_opts,
+        )
+        .map_err(|e| {
+            log::error!("failed to mount cliprdr fuse: {:?}", e);
+            CliprdrError::CliprdrInit
+        })?;
+        *fuse_handle = Some(new_handle);
+
+        let clipboard = self.clipboard.clone();
+
+        std::thread::spawn(move || {
+            log::debug!("start listening clipboard");
+            clipboard.start();
+        });
+
+        Ok(())
+    }
+
+    /// set clipboard data from file list
+    pub fn set_clipboard(&self, paths: &[PathBuf]) -> Result<(), CliprdrError> {
+        let prefix = self.fuse_mount_point.clone();
+        let paths: Vec<PathBuf> = paths.iter().cloned().map(|p| prefix.join(p)).collect();
+        log::debug!("setting clipboard with paths: {:?}", paths);
+        self.clipboard.set_file_list(&paths)?;
+        log::debug!("clipboard set, paths: {:?}", paths);
+        Ok(())
+    }
+
+    fn serve_file_contents(
+        &self,
+        conn_id: i32,
+        request: FileContentsRequest,
+    ) -> Result<(), CliprdrError> {
+        let mut file_list = self.local_files.lock();
+
+        let (file_idx, file_contents_resp) = match request {
+            FileContentsRequest::Size {
+                stream_id,
+                file_idx,
+            } => {
+                log::debug!("file contents (size) requested from conn: {}", conn_id);
+                let Some(file) = file_list.get(file_idx) else {
+                    log::error!(
+                        "invalid file index {} requested from conn: {}",
+                        file_idx,
+                        conn_id
+                    );
+                    resp_file_contents_fail(conn_id, stream_id);
+
+                    return Err(CliprdrError::InvalidRequest {
+                        description: format!(
+                            "invalid file index {} requested from conn: {}",
+                            file_idx, conn_id
+                        ),
+                    });
+                };
+
+                log::debug!(
+                    "conn {} requested file-{}: {}",
+                    conn_id,
+                    file_idx,
+                    file.name
+                );
+
+                let size = file.size;
+                (
+                    file_idx,
+                    ClipboardFile::FileContentsResponse {
+                        msg_flags: 0x1,
+                        stream_id,
+                        requested_data: size.to_le_bytes().to_vec(),
+                    },
+                )
+            }
+            FileContentsRequest::Range {
+                stream_id,
+                file_idx,
+                offset,
+                length,
+            } => {
+                log::debug!(
+                    "file contents (range from {} length {}) request from conn: {}",
+                    offset,
+                    length,
+                    conn_id
+                );
+                let Some(file) = file_list.get_mut(file_idx) else {
+                    log::error!(
+                        "invalid file index {} requested from conn: {}",
+                        file_idx,
+                        conn_id
+                    );
+                    resp_file_contents_fail(conn_id, stream_id);
+                    return Err(CliprdrError::InvalidRequest {
+                        description: format!(
+                            "invalid file index {} requested from conn: {}",
+                            file_idx, conn_id
+                        ),
+                    });
+                };
+                log::debug!(
+                    "conn {} requested file-{}: {}",
+                    conn_id,
+                    file_idx,
+                    file.name
+                );
+
+                if offset > file.size {
+                    log::error!("invalid reading offset requested from conn: {}", conn_id);
+                    resp_file_contents_fail(conn_id, stream_id);
+
+                    return Err(CliprdrError::InvalidRequest {
+                        description: format!(
+                            "invalid reading offset requested from conn: {}",
+                            conn_id
+                        ),
+                    });
+                }
+                let read_size = if offset + length > file.size {
+                    file.size - offset
+                } else {
+                    length
+                };
+
+                let mut buf = vec![0u8; read_size as usize];
+
+                file.read_exact_at(&mut buf, offset)?;
+
+                (
+                    file_idx,
+                    ClipboardFile::FileContentsResponse {
+                        msg_flags: 0x1,
+                        stream_id,
+                        requested_data: buf,
+                    },
+                )
+            }
+        };
+
+        send_data(conn_id, file_contents_resp);
+        log::debug!("file contents sent to conn: {}", conn_id);
+        // hot reload next file
+        for next_file in file_list.iter_mut().skip(file_idx + 1) {
+            if !next_file.is_dir {
+                next_file.load_handle()?;
+                break;
+            }
+        }
+        Ok(())
+    }
+}
+
+fn resp_file_contents_fail(conn_id: i32, stream_id: i32) {
+    let resp = ClipboardFile::FileContentsResponse {
+        msg_flags: 0x2,
+        stream_id,
+        requested_data: vec![],
+    };
+    send_data(conn_id, resp)
+}
+
+impl ClipboardContext {
+    pub fn is_stopped(&self) -> bool {
+        self.fuse_handle.lock().is_none()
+    }
+
+    pub fn sync_local_files(&self) -> Result<(), CliprdrError> {
+        let mut local_files = self.local_files.lock();
+        let clipboard_files = self.clipboard.get_file_list();
+        let local_file_list: Vec<PathBuf> = local_files.iter().map(|f| f.path.clone()).collect();
+        if local_file_list == clipboard_files {
+            return Ok(());
+        }
+        let new_files = construct_file_list(&clipboard_files)?;
+        *local_files = new_files;
+        Ok(())
+    }
+
+    pub fn serve(&self, conn_id: i32, msg: ClipboardFile) -> Result<(), CliprdrError> {
+        log::debug!("serve clipboard file from conn: {}", conn_id);
+        if self.is_stopped() {
+            log::debug!("cliprdr stopped, restart it");
+            self.run()?;
+        }
+        match msg {
+            ClipboardFile::NotifyCallback { .. } => {
+                unreachable!()
+            }
+            ClipboardFile::MonitorReady => {
+                log::debug!("server_monitor_ready called");
+
+                self.send_file_list(conn_id)?;
+
+                Ok(())
+            }
+
+            ClipboardFile::FormatList { format_list } => {
+                log::debug!("server_format_list called");
+                // filter out "FileGroupDescriptorW" and "FileContents"
+                let fmt_lst: Vec<(i32, String)> = format_list
+                    .into_iter()
+                    .filter(|(_, name)| {
+                        name == FILEDESCRIPTORW_FORMAT_NAME || name == FILECONTENTS_FORMAT_NAME
+                    })
+                    .collect();
+                if fmt_lst.len() != 2 {
+                    log::debug!("no supported formats");
+                    return Ok(());
+                }
+                log::debug!("supported formats: {:?}", fmt_lst);
+                let file_contents_id = fmt_lst
+                    .iter()
+                    .find(|(_, name)| name == FILECONTENTS_FORMAT_NAME)
+                    .map(|(id, _)| *id)
+                    .unwrap();
+                let file_descriptor_id = fmt_lst
+                    .iter()
+                    .find(|(_, name)| name == FILEDESCRIPTORW_FORMAT_NAME)
+                    .map(|(id, _)| *id)
+                    .unwrap();
+
+                add_remote_format(FILECONTENTS_FORMAT_NAME, file_contents_id);
+                add_remote_format(FILEDESCRIPTORW_FORMAT_NAME, file_descriptor_id);
+
+                // sync file system from peer
+                let data = ClipboardFile::FormatDataRequest {
+                    requested_format_id: file_descriptor_id,
+                };
+                send_data(conn_id, data);
+
+                Ok(())
+            }
+            ClipboardFile::FormatListResponse { msg_flags } => {
+                log::debug!("server_format_list_response called");
+                if msg_flags != 0x1 {
+                    send_format_list(conn_id)
+                } else {
+                    Ok(())
+                }
+            }
+            ClipboardFile::FormatDataRequest {
+                requested_format_id,
+            } => {
+                log::debug!("server_format_data_request called");
+                let Some(format) = get_local_format(requested_format_id) else {
+                    log::error!(
+                        "got unsupported format data request: id={} from conn={}",
+                        requested_format_id,
+                        conn_id
+                    );
+                    resp_format_data_failure(conn_id);
+                    return Ok(());
+                };
+
+                if format == FILEDESCRIPTORW_FORMAT_NAME {
+                    self.send_file_list(conn_id)?;
+                } else if format == FILECONTENTS_FORMAT_NAME {
+                    log::error!(
+                        "try to read file contents with FormatDataRequest from conn={}",
+                        conn_id
+                    );
+                    resp_format_data_failure(conn_id);
+                } else {
+                    log::error!(
+                        "got unsupported format data request: id={} from conn={}",
+                        requested_format_id,
+                        conn_id
+                    );
+                    resp_format_data_failure(conn_id);
+                }
+                Ok(())
+            }
+            ClipboardFile::FormatDataResponse {
+                msg_flags,
+                format_data,
+            } => {
+                log::debug!(
+                    "server_format_data_response called, msg_flags={}",
+                    msg_flags
+                );
+
+                if msg_flags != 0x1 {
+                    resp_format_data_failure(conn_id);
+                    return Ok(());
+                }
+
+                log::debug!("parsing file descriptors");
+                // this must be a file descriptor format data
+                let files = FileDescription::parse_file_descriptors(format_data, conn_id)?;
+
+                let paths = {
+                    let mut fuse_guard = self.fuse_server.lock();
+                    fuse_guard.load_file_list(files)?;
+
+                    fuse_guard.list_root()
+                };
+
+                log::debug!("load file list: {:?}", paths);
+                self.set_clipboard(&paths)?;
+                Ok(())
+            }
+            ClipboardFile::FileContentsResponse { .. } => {
+                log::debug!("server_file_contents_response called");
+                // we don't know its corresponding request, no resend can be performed
+                self.fuse_tx.send(msg).map_err(|e| {
+                    log::error!("failed to send file contents response to fuse: {:?}", e);
+                    CliprdrError::ClipboardInternalError
+                })?;
+                Ok(())
+            }
+            ClipboardFile::FileContentsRequest {
+                stream_id,
+                list_index,
+                dw_flags,
+                n_position_low,
+                n_position_high,
+                cb_requested,
+                ..
+            } => {
+                log::debug!("server_file_contents_request called");
+                let fcr = if dw_flags == 0x1 {
+                    FileContentsRequest::Size {
+                        stream_id,
+                        file_idx: list_index as usize,
+                    }
+                } else if dw_flags == 0x2 {
+                    let offset = (n_position_high as u64) << 32 | n_position_low as u64;
+                    let length = cb_requested as u64;
+
+                    FileContentsRequest::Range {
+                        stream_id,
+                        file_idx: list_index as usize,
+                        offset,
+                        length,
+                    }
+                } else {
+                    log::error!("got invalid FileContentsRequest from conn={}", conn_id);
+                    resp_file_contents_fail(conn_id, stream_id);
+                    return Ok(());
+                };
+
+                self.serve_file_contents(conn_id, fcr)
+            }
+        }
+    }
+
+    fn send_file_list(&self, conn_id: i32) -> Result<(), CliprdrError> {
+        self.sync_local_files()?;
+
+        let file_list = self.local_files.lock();
+        send_file_list(&*file_list, conn_id)
+    }
+}
+
+impl CliprdrServiceContext for ClipboardContext {
+    fn set_is_stopped(&mut self) -> Result<(), CliprdrError> {
+        // unmount the fuse
+        if let Some(fuse_handle) = self.fuse_handle.lock().take() {
+            fuse_handle.join();
+        }
+        // we don't stop the clipboard, keep listening in case of restart
+        Ok(())
+    }
+
+    fn empty_clipboard(&mut self, _conn_id: i32) -> Result<bool, CliprdrError> {
+        self.clipboard.set_file_list(&[])?;
+        Ok(true)
+    }
+
+    fn server_clip_file(&mut self, conn_id: i32, msg: ClipboardFile) -> Result<(), CliprdrError> {
+        self.serve(conn_id, msg)
+    }
+}
+
+fn resp_format_data_failure(conn_id: i32) {
+    let data = ClipboardFile::FormatDataResponse {
+        msg_flags: 0x2,
+        format_data: vec![],
+    };
+    send_data(conn_id, data)
+}
+
+fn send_format_list(conn_id: i32) -> Result<(), CliprdrError> {
+    log::debug!("send format list to remote, conn={}", conn_id);
+    let fd_format_name = get_local_format(FILEDESCRIPTOR_FORMAT_ID)
+        .unwrap_or(FILEDESCRIPTORW_FORMAT_NAME.to_string());
+    let fc_format_name =
+        get_local_format(FILECONTENTS_FORMAT_ID).unwrap_or(FILECONTENTS_FORMAT_NAME.to_string());
+    let format_list = ClipboardFile::FormatList {
+        format_list: vec![
+            (FILEDESCRIPTOR_FORMAT_ID, fd_format_name),
+            (FILECONTENTS_FORMAT_ID, fc_format_name),
+        ],
+    };
+
+    send_data(conn_id, format_list);
+    log::debug!("format list to remote dispatched, conn={}", conn_id);
+    Ok(())
+}
+
+fn build_file_list_pdu(files: &[LocalFile]) -> Vec<u8> {
+    let mut data = BytesMut::with_capacity(4 + 592 * files.len());
+    data.put_u32_le(files.len() as u32);
+    for file in files.iter() {
+        data.put(file.as_bin().as_slice());
+    }
+
+    data.to_vec()
+}
+
+fn send_file_list(files: &[LocalFile], conn_id: i32) -> Result<(), CliprdrError> {
+    log::debug!(
+        "send file list to remote, conn={}, list={:?}",
+        conn_id,
+        files.iter().map(|f| f.path.display()).collect::<Vec<_>>()
+    );
+
+    let format_data = build_file_list_pdu(files);
+
+    send_data(
+        conn_id,
+        ClipboardFile::FormatDataResponse {
+            msg_flags: 1,
+            format_data,
+        },
+    );
+    Ok(())
+}

libs/clipboard/src/platform/unix/ns_clipboard.rs

@@ -0,0 +1,97 @@
+use std::{collections::BTreeSet, path::PathBuf};
+
+use cacao::pasteboard::{Pasteboard, PasteboardName};
+use hbb_common::log;
+use parking_lot::Mutex;
+
+use crate::{platform::unix::send_format_list, CliprdrError};
+
+use super::SysClipboard;
+
+#[inline]
+fn wait_file_list() -> Option<Vec<PathBuf>> {
+    let pb = Pasteboard::named(PasteboardName::General);
+    pb.get_file_urls()
+        .ok()
+        .map(|v| v.into_iter().map(|nsurl| nsurl.pathbuf()).collect())
+}
+
+#[inline]
+fn set_file_list(file_list: &[PathBuf]) -> Result<(), CliprdrError> {
+    let pb = Pasteboard::named(PasteboardName::General);
+    pb.set_files(file_list.to_vec())
+        .map_err(|_| CliprdrError::ClipboardInternalError)
+}
+
+pub struct NsPasteboard {
+    ignore_path: PathBuf,
+
+    former_file_list: Mutex<Vec<PathBuf>>,
+}
+
+impl NsPasteboard {
+    pub fn new(ignore_path: &PathBuf) -> Result<Self, CliprdrError> {
+        Ok(Self {
+            ignore_path: ignore_path.to_owned(),
+            former_file_list: Mutex::new(vec![]),
+        })
+    }
+}
+
+impl SysClipboard for NsPasteboard {
+    fn set_file_list(&self, paths: &[PathBuf]) -> Result<(), CliprdrError> {
+        *self.former_file_list.lock() = paths.to_vec();
+        set_file_list(paths)
+    }
+
+    fn start(&self) {
+        {
+            *self.former_file_list.lock() = vec![];
+        }
+
+        loop {
+            let file_list = match wait_file_list() {
+                Some(v) => v,
+                None => {
+                    std::thread::sleep(std::time::Duration::from_millis(100));
+                    continue;
+                }
+            };
+
+            let filtered = file_list
+                .into_iter()
+                .filter(|pb| !pb.starts_with(&self.ignore_path))
+                .collect::<Vec<_>>();
+
+            if filtered.is_empty() {
+                std::thread::sleep(std::time::Duration::from_millis(100));
+                continue;
+            }
+
+            {
+                let mut former = self.former_file_list.lock();
+
+                let filtered_st: BTreeSet<_> = filtered.iter().collect();
+                let former_st = former.iter().collect::<BTreeSet<_>>();
+                if filtered_st == former_st {
+                    std::thread::sleep(std::time::Duration::from_millis(100));
+                    continue;
+                }
+
+                *former = filtered;
+            }
+
+            if let Err(e) = send_format_list(0) {
+                log::warn!("failed to send format list: {}", e);
+                break;
+            }
+
+            std::thread::sleep(std::time::Duration::from_millis(100));
+        }
+        log::debug!("stop listening file related atoms on clipboard");
+    }
+
+    fn get_file_list(&self) -> Vec<PathBuf> {
+        self.former_file_list.lock().clone()
+    }
+}

libs/clipboard/src/platform/unix/url.rs

@@ -0,0 +1,75 @@
+use std::path::{Path, PathBuf};
+
+use crate::CliprdrError;
+
+// on x11, path will be encode as
+// "/home/rustdesk/pictures/🖼️.png" -> "file:///home/rustdesk/pictures/%F0%9F%96%BC%EF%B8%8F.png"
+// url encode and decode is needed
+const ENCODE_SET: percent_encoding::AsciiSet = percent_encoding::CONTROLS.add(b' ').remove(b'/');
+
+pub(super) fn encode_path_to_uri(path: &PathBuf) -> String {
+    let encoded = percent_encoding::percent_encode(path.to_str().unwrap().as_bytes(), &ENCODE_SET)
+        .to_string();
+    format!("file://{}", encoded)
+}
+
+pub(super) fn parse_uri_to_path(encoded_uri: &str) -> Result<PathBuf, CliprdrError> {
+    let encoded_path = encoded_uri.trim_start_matches("file://");
+    let path_str = percent_encoding::percent_decode_str(encoded_path)
+        .decode_utf8()
+        .map_err(|_| CliprdrError::ConversionFailure)?;
+    let path_str = path_str.to_string();
+
+    Ok(Path::new(&path_str).to_path_buf())
+}
+
+// helper parse function
+// convert 'text/uri-list' data to a list of valid Paths
+// # Note
+// - none utf8 data will lead to error
+pub(super) fn parse_plain_uri_list(v: Vec<u8>) -> Result<Vec<PathBuf>, CliprdrError> {
+    let text = String::from_utf8(v).map_err(|_| CliprdrError::ConversionFailure)?;
+    parse_uri_list(&text)
+}
+
+// helper parse function
+// convert 'text/uri-list' data to a list of valid Paths
+// # Note
+// - none utf8 data will lead to error
+pub(super) fn parse_uri_list(text: &str) -> Result<Vec<PathBuf>, CliprdrError> {
+    let mut list = Vec::new();
+
+    for line in text.lines() {
+        if !line.starts_with("file://") {
+            continue;
+        }
+        let decoded = parse_uri_to_path(line)?;
+        list.push(decoded)
+    }
+    Ok(list)
+}
+
+#[cfg(test)]
+mod uri_test {
+    #[test]
+    fn test_conversion() {
+        let path = std::path::PathBuf::from("/home/rustdesk/pictures/🖼️.png");
+        let uri = super::encode_path_to_uri(&path);
+        assert_eq!(
+            uri,
+            "file:///home/rustdesk/pictures/%F0%9F%96%BC%EF%B8%8F.png"
+        );
+        let convert_back = super::parse_uri_to_path(&uri).unwrap();
+        assert_eq!(path, convert_back);
+    }
+
+    #[test]
+    fn parse_list() {
+        let uri_list = r#"file:///home/rustdesk/pictures/%F0%9F%96%BC%EF%B8%8F.png
+file:///home/rustdesk/pictures/%F0%9F%96%BC%EF%B8%8F.png
+"#;
+        let list = super::parse_uri_list(uri_list.into()).unwrap();
+        assert!(list.len() == 2);
+        assert_eq!(list[0], list[1]);
+    }
+}

libs/clipboard/src/platform/unix/x11.rs

@@ -0,0 +1,162 @@
+use std::{collections::BTreeSet, path::PathBuf};
+
+use hbb_common::log;
+use once_cell::sync::OnceCell;
+use parking_lot::Mutex;
+use x11_clipboard::Clipboard;
+use x11rb::protocol::xproto::Atom;
+
+use crate::{platform::unix::send_format_list, CliprdrError};
+
+use super::{encode_path_to_uri, parse_plain_uri_list, SysClipboard};
+
+static X11_CLIPBOARD: OnceCell<Clipboard> = OnceCell::new();
+
+fn get_clip() -> Result<&'static Clipboard, CliprdrError> {
+    X11_CLIPBOARD.get_or_try_init(|| Clipboard::new().map_err(|_| CliprdrError::CliprdrInit))
+}
+
+pub struct X11Clipboard {
+    ignore_path: PathBuf,
+    text_uri_list: Atom,
+    gnome_copied_files: Atom,
+    nautilus_clipboard: Atom,
+
+    former_file_list: Mutex<Vec<PathBuf>>,
+}
+
+impl X11Clipboard {
+    pub fn new(ignore_path: &PathBuf) -> Result<Self, CliprdrError> {
+        let clipboard = get_clip()?;
+        let text_uri_list = clipboard
+            .setter
+            .get_atom("text/uri-list")
+            .map_err(|_| CliprdrError::CliprdrInit)?;
+        let gnome_copied_files = clipboard
+            .setter
+            .get_atom("x-special/gnome-copied-files")
+            .map_err(|_| CliprdrError::CliprdrInit)?;
+        let nautilus_clipboard = clipboard
+            .setter
+            .get_atom("x-special/nautilus-clipboard")
+            .map_err(|_| CliprdrError::CliprdrInit)?;
+        Ok(Self {
+            ignore_path: ignore_path.to_owned(),
+            text_uri_list,
+            gnome_copied_files,
+            nautilus_clipboard,
+            former_file_list: Mutex::new(vec![]),
+        })
+    }
+
+    fn load(&self, target: Atom) -> Result<Vec<u8>, CliprdrError> {
+        let clip = get_clip()?.setter.atoms.clipboard;
+        let prop = get_clip()?.setter.atoms.property;
+        // NOTE:
+        // # why not use `load_wait`
+        // load_wait is likely to wait forever, which is not what we want
+        let res = get_clip()?.load_wait(clip, target, prop);
+        match res {
+            Ok(res) => Ok(res),
+            Err(x11_clipboard::error::Error::UnexpectedType(_)) => Ok(vec![]),
+            Err(x11_clipboard::error::Error::Timeout) => {
+                log::debug!("x11 clipboard get content timeout.");
+                Err(CliprdrError::ClipboardInternalError)
+            }
+            Err(e) => {
+                log::debug!("x11 clipboard get content fail: {:?}", e);
+                Err(CliprdrError::ClipboardInternalError)
+            }
+        }
+    }
+
+    fn store_batch(&self, batch: Vec<(Atom, Vec<u8>)>) -> Result<(), CliprdrError> {
+        let clip = get_clip()?.setter.atoms.clipboard;
+        log::debug!("try to store clipboard content");
+        get_clip()?
+            .store_batch(clip, batch)
+            .map_err(|_| CliprdrError::ClipboardInternalError)
+    }
+
+    fn wait_file_list(&self) -> Result<Option<Vec<PathBuf>>, CliprdrError> {
+        let v = self.load(self.text_uri_list)?;
+        let p = parse_plain_uri_list(v)?;
+        Ok(Some(p))
+    }
+}
+
+impl SysClipboard for X11Clipboard {
+    fn set_file_list(&self, paths: &[PathBuf]) -> Result<(), CliprdrError> {
+        *self.former_file_list.lock() = paths.to_vec();
+
+        let uri_list: Vec<String> = paths.iter().map(encode_path_to_uri).collect();
+        let uri_list = uri_list.join("\n");
+        let text_uri_list_data = uri_list.as_bytes().to_vec();
+        let gnome_copied_files_data = ["copy\n".as_bytes(), uri_list.as_bytes()].concat();
+        let batch = vec![
+            (self.text_uri_list, text_uri_list_data),
+            (self.gnome_copied_files, gnome_copied_files_data.clone()),
+            (self.nautilus_clipboard, gnome_copied_files_data),
+        ];
+        self.store_batch(batch)
+            .map_err(|_| CliprdrError::ClipboardInternalError)
+    }
+
+    fn start(&self) {
+        {
+            // clear cached file list
+            *self.former_file_list.lock() = vec![];
+        }
+        loop {
+            let sth = match self.wait_file_list() {
+                Ok(sth) => sth,
+                Err(e) => {
+                    log::warn!("failed to get file list from clipboard: {}", e);
+                    std::thread::sleep(std::time::Duration::from_millis(100));
+                    continue;
+                }
+            };
+
+            let Some(paths) = sth else {
+                // just sleep
+                std::thread::sleep(std::time::Duration::from_millis(100));
+                continue;
+            };
+
+            let filtered = paths
+                .into_iter()
+                .filter(|pb| !pb.starts_with(&self.ignore_path))
+                .collect::<Vec<_>>();
+
+            if filtered.is_empty() {
+                std::thread::sleep(std::time::Duration::from_millis(100));
+                continue;
+            }
+
+            {
+                let mut former = self.former_file_list.lock();
+
+                let filtered_st: BTreeSet<_> = filtered.iter().collect();
+                let former_st = former.iter().collect::<BTreeSet<_>>();
+                if filtered_st == former_st {
+                    std::thread::sleep(std::time::Duration::from_millis(100));
+                    continue;
+                }
+
+                *former = filtered;
+            }
+
+            if let Err(e) = send_format_list(0) {
+                log::warn!("failed to send format list: {}", e);
+                break;
+            }
+
+            std::thread::sleep(std::time::Duration::from_millis(100));
+        }
+        log::debug!("stop listening file related atoms on clipboard");
+    }
+
+    fn get_file_list(&self) -> Vec<PathBuf> {
+        self.former_file_list.lock().clone()
+    }
+}

libs/hbb_common/Cargo.toml

@@ -9,7 +9,7 @@ edition = "2018"
 [dependencies]
 flexi_logger = { version = "0.25", features = ["async"] }
 protobuf = { version = "3.2", features = ["with-bytes"] }
-tokio = { version = "1.28", features = ["full"] }
+tokio = { version = "=1.28.1", features = ["full"] }
 tokio-util = { version = "0.7", features = ["full"] }
 futures = "0.3"
 bytes = { version = "1.4", features = ["serde"] }

libs/hbb_common/protos/message.proto

@@ -17,6 +17,11 @@ message YUV {
   int32 stride = 2;
 }
 
+enum Chroma {
+  I420 = 0;
+  I444 = 1;
+}
+
 message VideoFrame {
   oneof union {
     EncodedVideoFrames vp9s = 6;
@@ -83,11 +88,20 @@ message Features {
   bool privacy_mode = 1;
 }
 
+message CodecAbility {
+  bool vp8 = 1;
+  bool vp9 = 2;
+  bool av1 = 3;
+  bool h264 = 4;
+  bool h265 = 5;
+}
+
 message SupportedEncoding {
   bool h264 = 1;
   bool h265 = 2;
   bool vp8 = 3;
   bool av1 = 4;
+  CodecAbility i444 = 5;
 }
 
 message PeerInfo {
@@ -512,6 +526,7 @@ message PermissionInfo {
     File = 4;
     Restart = 5;
     Recording = 6;
+    BlockInput = 7;
   }
 
   Permission permission = 1;
@@ -541,6 +556,8 @@ message SupportedDecoding {
   PreferCodec prefer = 4;
   int32 ability_vp8 = 5;
   int32 ability_av1 = 6;
+  CodecAbility i444 = 7;
+  Chroma prefer_chroma = 8;
 }
 
 message OptionMessage {

libs/hbb_common/src/config.rs

@@ -91,10 +91,11 @@ const CHARS: &[char] = &[
 ];
 
 pub const RENDEZVOUS_SERVERS: &[&str] = &["rs-ny.rustdesk.com"];
+pub const PUBLIC_RS_PUB_KEY: &str = "OeVuKk5nlHiXp+APNn0Y3pC1Iwpwn44JGqrQCsWqmBw=";
 
 pub const RS_PUB_KEY: &str = match option_env!("RS_PUB_KEY") {
     Some(key) if !key.is_empty() => key,
-    _ => "OeVuKk5nlHiXp+APNn0Y3pC1Iwpwn44JGqrQCsWqmBw=",
+    _ => PUBLIC_RS_PUB_KEY,
 };
 
 pub const RENDEZVOUS_PORT: i32 = 21116;
@@ -1229,6 +1230,10 @@ impl PeerConfig {
         if !mp.contains_key(key) {
             mp.insert(key.to_owned(), UserDefaultConfig::read().get(key));
         }
+        key = "i444";
+        if !mp.contains_key(key) {
+            mp.insert(key.to_owned(), UserDefaultConfig::read().get(key));
+        }
     }
 }
 

libs/portable/generate.py

@@ -1,4 +1,3 @@
-from ast import parse
 import os
 import optparse
 from hashlib import md5
@@ -47,7 +46,7 @@ def write_metadata(md5_table: dict, output_folder: str, exe: str):
             f.write((len(path)).to_bytes(length=length_count, byteorder='big'))
             f.write(path)
             # data length & compressed data
-            f.write((data_length).to_bytes(
+            f.write(data_length.to_bytes(
                 length=length_count, byteorder='big'))
             f.write(compressed_data)
             # md5 code
@@ -65,6 +64,8 @@ def build_portable(output_folder: str):
 
 # Linux: python3 generate.py -f ../rustdesk-portable-packer/test -o . -e ./test/main.py
 # Windows: python3 .\generate.py -f ..\rustdesk\flutter\build\windows\runner\Debug\ -o . -e ..\rustdesk\flutter\build\windows\runner\Debug\rustdesk.exe
+
+
 if __name__ == '__main__':
     parser = optparse.OptionParser()
     parser.add_option("-f", "--folder", dest="folder",

libs/scrap/build.rs

@@ -197,6 +197,7 @@ fn main() {
     find_package("libyuv");
     gen_vcpkg_package("libvpx", "vpx_ffi.h", "vpx_ffi.rs", "^[vV].*");
     gen_vcpkg_package("aom", "aom_ffi.h", "aom_ffi.rs", "^(aom|AOM|OBU|AV1).*");
+    gen_vcpkg_package("libyuv", "yuv_ffi.h", "yuv_ffi.rs", ".*");
 
     // there is problem with cfg(target_os) in build.rs, so use our workaround
     let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap();

libs/scrap/examples/benchmark.rs

@@ -1,13 +1,20 @@
 use docopt::Docopt;
-use hbb_common::env_logger::{init_from_env, Env, DEFAULT_FILTER_ENV};
+use hbb_common::{
+    env_logger::{init_from_env, Env, DEFAULT_FILTER_ENV},
+    log,
+};
 use scrap::{
     aom::{AomDecoder, AomEncoder, AomEncoderConfig},
     codec::{EncoderApi, EncoderCfg, Quality as Q},
-    Capturer, Display, TraitCapturer, VpxDecoder, VpxDecoderConfig, VpxEncoder, VpxEncoderConfig,
+    convert_to_yuv, Capturer, Display, TraitCapturer, VpxDecoder, VpxDecoderConfig, VpxEncoder,
+    VpxEncoderConfig,
     VpxVideoCodecId::{self, *},
     STRIDE_ALIGN,
 };
-use std::{io::Write, time::Instant};
+use std::{
+    io::Write,
+    time::{Duration, Instant},
+};
 
 // cargo run --package scrap --example benchmark --release --features hwcodec
 
@@ -15,7 +22,7 @@ const USAGE: &'static str = "
 Codec benchmark.
 
 Usage:
-  benchmark [--count=COUNT] [--quality=QUALITY] [--hw-pixfmt=PIXFMT]
+  benchmark [--count=COUNT] [--quality=QUALITY] [--i444]
   benchmark (-h | --help)
 
 Options:
@@ -23,24 +30,17 @@ Options:
   --count=COUNT         Capture frame count [default: 100].
   --quality=QUALITY     Video quality [default: Balanced].
                         Valid values: Best, Balanced, Low.
-  --hw-pixfmt=PIXFMT    Hardware codec pixfmt. [default: i420]
-                        Valid values: i420, nv12.
+  --i444                I444.
 ";
 
-#[derive(Debug, serde::Deserialize)]
+#[derive(Debug, serde::Deserialize, Clone, Copy)]
 struct Args {
     flag_count: usize,
     flag_quality: Quality,
-    flag_hw_pixfmt: Pixfmt,
+    flag_i444: bool,
 }
 
-#[derive(Debug, serde::Deserialize)]
-enum Pixfmt {
-    I420,
-    NV12,
-}
-
-#[derive(Debug, serde::Deserialize)]
+#[derive(Debug, serde::Deserialize, Clone, Copy)]
 enum Quality {
     Best,
     Balanced,
@@ -54,31 +54,6 @@ fn main() {
         .unwrap_or_else(|e| e.exit());
     let quality = args.flag_quality;
     let yuv_count = args.flag_count;
-    let (yuvs, width, height) = capture_yuv(yuv_count);
-    println!(
-        "benchmark {}x{} quality:{:?}k hw_pixfmt:{:?}",
-        width, height, quality, args.flag_hw_pixfmt
-    );
-    let quality = match quality {
-        Quality::Best => Q::Best,
-        Quality::Balanced => Q::Balanced,
-        Quality::Low => Q::Low,
-    };
-    [VP8, VP9].map(|c| test_vpx(c, &yuvs, width, height, quality, yuv_count));
-    test_av1(&yuvs, width, height, quality, yuv_count);
-    #[cfg(feature = "hwcodec")]
-    {
-        use hwcodec::AVPixelFormat;
-        let hw_pixfmt = match args.flag_hw_pixfmt {
-            Pixfmt::I420 => AVPixelFormat::AV_PIX_FMT_YUV420P,
-            Pixfmt::NV12 => AVPixelFormat::AV_PIX_FMT_NV12,
-        };
-        let yuvs = hw::vpx_yuv_to_hw_yuv(yuvs, width, height, hw_pixfmt);
-        hw::test(&yuvs, width, height, quality, yuv_count, hw_pixfmt);
-    }
-}
-
-fn capture_yuv(yuv_count: usize) -> (Vec<Vec<u8>>, usize, usize) {
     let mut index = 0;
     let mut displays = Display::all().unwrap();
     for i in 0..displays.len() {
@@ -88,28 +63,45 @@ fn capture_yuv(yuv_count: usize) -> (Vec<Vec<u8>>, usize, usize) {
         }
     }
     let d = displays.remove(index);
-    let mut c = Capturer::new(d, true).unwrap();
-    let mut v = vec![];
-    loop {
-        if let Ok(frame) = c.frame(std::time::Duration::from_millis(30)) {
-            v.push(frame.0.to_vec());
-            print!("\rcapture {}/{}", v.len(), yuv_count);
-            std::io::stdout().flush().ok();
-            if v.len() == yuv_count {
-                println!();
-                return (v, c.width(), c.height());
-            }
-        }
+    let mut c = Capturer::new(d).unwrap();
+    let width = c.width();
+    let height = c.height();
+
+    println!(
+        "benchmark {}x{} quality:{:?}, i444:{:?}",
+        width, height, quality, args.flag_i444
+    );
+    let quality = match quality {
+        Quality::Best => Q::Best,
+        Quality::Balanced => Q::Balanced,
+        Quality::Low => Q::Low,
+    };
+    [VP8, VP9].map(|codec| {
+        test_vpx(
+            &mut c,
+            codec,
+            width,
+            height,
+            quality,
+            yuv_count,
+            if codec == VP8 { false } else { args.flag_i444 },
+        )
+    });
+    test_av1(&mut c, width, height, quality, yuv_count, args.flag_i444);
+    #[cfg(feature = "hwcodec")]
+    {
+        hw::test(&mut c, width, height, quality, yuv_count);
     }
 }
 
 fn test_vpx(
+    c: &mut Capturer,
     codec_id: VpxVideoCodecId,
-    yuvs: &Vec<Vec<u8>>,
     width: usize,
     height: usize,
     quality: Q,
     yuv_count: usize,
+    i444: bool,
 ) {
     let config = EncoderCfg::VPX(VpxEncoderConfig {
         width: width as _,
@@ -118,28 +110,53 @@ fn test_vpx(
         codec: codec_id,
         keyframe_interval: None,
     });
-    let mut encoder = VpxEncoder::new(config).unwrap();
+    let mut encoder = VpxEncoder::new(config, i444).unwrap();
     let mut vpxs = vec![];
     let start = Instant::now();
     let mut size = 0;
-    for yuv in yuvs {
-        for ref frame in encoder
-            .encode(start.elapsed().as_millis() as _, yuv, STRIDE_ALIGN)
-            .unwrap()
-        {
-            size += frame.data.len();
-            vpxs.push(frame.data.to_vec());
+    let mut yuv = Vec::new();
+    let mut mid_data = Vec::new();
+    let mut counter = 0;
+    let mut time_sum = Duration::ZERO;
+    loop {
+        match c.frame(std::time::Duration::from_millis(30)) {
+            Ok(frame) => {
+                let tmp_timer = Instant::now();
+                convert_to_yuv(&frame, encoder.yuvfmt(), &mut yuv, &mut mid_data);
+                for ref frame in encoder
+                    .encode(start.elapsed().as_millis() as _, &yuv, STRIDE_ALIGN)
+                    .unwrap()
+                {
+                    size += frame.data.len();
+                    vpxs.push(frame.data.to_vec());
+                    counter += 1;
+                    print!("\r{codec_id:?} {}/{}", counter, yuv_count);
+                    std::io::stdout().flush().ok();
+                }
+                for ref frame in encoder.flush().unwrap() {
+                    size += frame.data.len();
+                    vpxs.push(frame.data.to_vec());
+                    counter += 1;
+                    print!("\r{codec_id:?} {}/{}", counter, yuv_count);
+                    std::io::stdout().flush().ok();
+                }
+                time_sum += tmp_timer.elapsed();
+            }
+            Err(e) => {
+                log::error!("{e:?}");
+            }
         }
-        for ref frame in encoder.flush().unwrap() {
-            size += frame.data.len();
-            vpxs.push(frame.data.to_vec());
+        if counter >= yuv_count {
+            println!();
+            break;
         }
     }
+
     assert_eq!(vpxs.len(), yuv_count);
     println!(
         "{:?} encode: {:?}, {} byte",
         codec_id,
-        start.elapsed() / yuv_count as _,
+        time_sum / yuv_count as _,
         size / yuv_count
     );
 
@@ -156,30 +173,58 @@ fn test_vpx(
     );
 }
 
-fn test_av1(yuvs: &Vec<Vec<u8>>, width: usize, height: usize, quality: Q, yuv_count: usize) {
+fn test_av1(
+    c: &mut Capturer,
+    width: usize,
+    height: usize,
+    quality: Q,
+    yuv_count: usize,
+    i444: bool,
+) {
     let config = EncoderCfg::AOM(AomEncoderConfig {
         width: width as _,
         height: height as _,
         quality,
         keyframe_interval: None,
     });
-    let mut encoder = AomEncoder::new(config).unwrap();
+    let mut encoder = AomEncoder::new(config, i444).unwrap();
     let start = Instant::now();
     let mut size = 0;
-    let mut av1s = vec![];
-    for yuv in yuvs {
-        for ref frame in encoder
-            .encode(start.elapsed().as_millis() as _, yuv, STRIDE_ALIGN)
-            .unwrap()
-        {
-            size += frame.data.len();
-            av1s.push(frame.data.to_vec());
+    let mut av1s: Vec<Vec<u8>> = vec![];
+    let mut yuv = Vec::new();
+    let mut mid_data = Vec::new();
+    let mut counter = 0;
+    let mut time_sum = Duration::ZERO;
+    loop {
+        match c.frame(std::time::Duration::from_millis(30)) {
+            Ok(frame) => {
+                let tmp_timer = Instant::now();
+                convert_to_yuv(&frame, encoder.yuvfmt(), &mut yuv, &mut mid_data);
+                for ref frame in encoder
+                    .encode(start.elapsed().as_millis() as _, &yuv, STRIDE_ALIGN)
+                    .unwrap()
+                {
+                    size += frame.data.len();
+                    av1s.push(frame.data.to_vec());
+                    counter += 1;
+                    print!("\rAV1 {}/{}", counter, yuv_count);
+                    std::io::stdout().flush().ok();
+                }
+                time_sum += tmp_timer.elapsed();
+            }
+            Err(e) => {
+                log::error!("{e:?}");
+            }
+        }
+        if counter >= yuv_count {
+            println!();
+            break;
         }
     }
     assert_eq!(av1s.len(), yuv_count);
     println!(
         "AV1 encode: {:?}, {} byte",
-        start.elapsed() / yuv_count as _,
+        time_sum / yuv_count as _,
         size / yuv_count
     );
     let mut decoder = AomDecoder::new().unwrap();
@@ -193,165 +238,101 @@ fn test_av1(yuvs: &Vec<Vec<u8>>, width: usize, height: usize, quality: Q, yuv_co
 
 #[cfg(feature = "hwcodec")]
 mod hw {
-    use super::*;
-    use hwcodec::{
-        decode::{DecodeContext, Decoder},
-        encode::{EncodeContext, Encoder},
-        ffmpeg::{ffmpeg_linesize_offset_length, CodecInfo, CodecInfos},
-        AVPixelFormat,
-        Quality::*,
-        RateControl::*,
-    };
+    use hwcodec::ffmpeg::CodecInfo;
     use scrap::{
-        codec::codec_thread_num,
-        convert::{
-            hw::{hw_bgra_to_i420, hw_bgra_to_nv12},
-            i420_to_bgra,
-        },
-        HW_STRIDE_ALIGN,
+        codec::HwEncoderConfig,
+        hwcodec::{HwDecoder, HwEncoder},
     };
 
-    pub fn test(
-        yuvs: &Vec<Vec<u8>>,
+    use super::*;
+
+    pub fn test(c: &mut Capturer, width: usize, height: usize, quality: Q, yuv_count: usize) {
+        let best = HwEncoder::best();
+        let mut h264s = Vec::new();
+        let mut h265s = Vec::new();
+        if let Some(info) = best.h264 {
+            test_encoder(width, height, quality, info, c, yuv_count, &mut h264s);
+        }
+        if let Some(info) = best.h265 {
+            test_encoder(width, height, quality, info, c, yuv_count, &mut h265s);
+        }
+        let best = HwDecoder::best();
+        if let Some(info) = best.h264 {
+            test_decoder(info, &h264s);
+        }
+        if let Some(info) = best.h265 {
+            test_decoder(info, &h265s);
+        }
+    }
+
+    fn test_encoder(
         width: usize,
         height: usize,
         quality: Q,
+        info: CodecInfo,
+        c: &mut Capturer,
         yuv_count: usize,
-        pixfmt: AVPixelFormat,
+        h26xs: &mut Vec<Vec<u8>>,
     ) {
-        let bitrate = scrap::hwcodec::HwEncoder::convert_quality(quality);
-        let ctx = EncodeContext {
-            name: String::from(""),
-            width: width as _,
-            height: height as _,
-            pixfmt,
-            align: 0,
-            bitrate: bitrate as i32 * 1000,
-            timebase: [1, 30],
-            gop: 60,
-            quality: Quality_Default,
-            rc: RC_DEFAULT,
-            thread_count: codec_thread_num() as _,
-        };
-
-        let encoders = Encoder::available_encoders(ctx.clone());
-        println!("hw encoders: {}", encoders.len());
-        let best = CodecInfo::score(encoders.clone());
-        for info in encoders {
-            test_encoder(info.clone(), ctx.clone(), yuvs, is_best(&best, &info));
-        }
-
-        let (h264s, h265s) = prepare_h26x(best, ctx.clone(), yuvs);
-        assert!(h264s.is_empty() || h264s.len() == yuv_count);
-        assert!(h265s.is_empty() || h265s.len() == yuv_count);
-        let decoders = Decoder::available_decoders();
-        println!("hw decoders: {}", decoders.len());
-        let best = CodecInfo::score(decoders.clone());
-        for info in decoders {
-            let h26xs = if info.name.contains("h264") {
-                &h264s
-            } else {
-                &h265s
-            };
-            if h26xs.len() == yuvs.len() {
-                test_decoder(info.clone(), h26xs, is_best(&best, &info));
-            }
-        }
-    }
-
-    fn test_encoder(info: CodecInfo, ctx: EncodeContext, yuvs: &Vec<Vec<u8>>, best: bool) {
-        let mut ctx = ctx;
-        ctx.name = info.name;
-        let mut encoder = Encoder::new(ctx.clone()).unwrap();
-        let start = Instant::now();
+        let mut encoder = HwEncoder::new(
+            EncoderCfg::HW(HwEncoderConfig {
+                name: info.name.clone(),
+                width,
+                height,
+                quality,
+                keyframe_interval: None,
+            }),
+            false,
+        )
+        .unwrap();
         let mut size = 0;
-        for yuv in yuvs {
-            let frames = encoder.encode(yuv).unwrap();
-            for frame in frames {
-                size += frame.data.len();
+
+        let mut yuv = Vec::new();
+        let mut mid_data = Vec::new();
+        let mut counter = 0;
+        let mut time_sum = Duration::ZERO;
+        loop {
+            match c.frame(std::time::Duration::from_millis(30)) {
+                Ok(frame) => {
+                    let tmp_timer = Instant::now();
+                    convert_to_yuv(&frame, encoder.yuvfmt(), &mut yuv, &mut mid_data);
+                    for ref frame in encoder.encode(&yuv).unwrap() {
+                        size += frame.data.len();
+
+                        h26xs.push(frame.data.to_vec());
+                        counter += 1;
+                        print!("\r{:?} {}/{}", info.name, counter, yuv_count);
+                        std::io::stdout().flush().ok();
+                    }
+                    time_sum += tmp_timer.elapsed();
+                }
+                Err(e) => {
+                    log::error!("{e:?}");
+                }
+            }
+            if counter >= yuv_count {
+                println!();
+                break;
             }
         }
         println!(
-            "{}{}: {:?}, {} byte",
-            if best { "*" } else { "" },
-            ctx.name,
-            start.elapsed() / yuvs.len() as _,
-            size / yuvs.len(),
+            "{}: {:?}, {} byte",
+            info.name,
+            time_sum / yuv_count as u32,
+            size / yuv_count,
         );
     }
 
-    fn test_decoder(info: CodecInfo, h26xs: &Vec<Vec<u8>>, best: bool) {
-        let ctx = DecodeContext {
-            name: info.name,
-            device_type: info.hwdevice,
-            thread_count: codec_thread_num() as _,
-        };
-
-        let mut decoder = Decoder::new(ctx.clone()).unwrap();
+    fn test_decoder(info: CodecInfo, h26xs: &Vec<Vec<u8>>) {
+        let mut decoder = HwDecoder::new(info.clone()).unwrap();
         let start = Instant::now();
         let mut cnt = 0;
         for h26x in h26xs {
             let _ = decoder.decode(h26x).unwrap();
             cnt += 1;
         }
-        let device = format!("{:?}", ctx.device_type).to_lowercase();
+        let device = format!("{:?}", info.hwdevice).to_lowercase();
         let device = device.split("_").last().unwrap();
-        println!(
-            "{}{} {}: {:?}",
-            if best { "*" } else { "" },
-            ctx.name,
-            device,
-            start.elapsed() / cnt
-        );
-    }
-
-    fn prepare_h26x(
-        best: CodecInfos,
-        ctx: EncodeContext,
-        yuvs: &Vec<Vec<u8>>,
-    ) -> (Vec<Vec<u8>>, Vec<Vec<u8>>) {
-        let f = |info: Option<CodecInfo>| {
-            let mut h26xs = vec![];
-            if let Some(info) = info {
-                let mut ctx = ctx.clone();
-                ctx.name = info.name;
-                let mut encoder = Encoder::new(ctx).unwrap();
-                for yuv in yuvs {
-                    let h26x = encoder.encode(yuv).unwrap();
-                    for frame in h26x {
-                        h26xs.push(frame.data.to_vec());
-                    }
-                }
-            }
-            h26xs
-        };
-        (f(best.h264), f(best.h265))
-    }
-
-    fn is_best(best: &CodecInfos, info: &CodecInfo) -> bool {
-        Some(info.clone()) == best.h264 || Some(info.clone()) == best.h265
-    }
-
-    pub fn vpx_yuv_to_hw_yuv(
-        yuvs: Vec<Vec<u8>>,
-        width: usize,
-        height: usize,
-        pixfmt: AVPixelFormat,
-    ) -> Vec<Vec<u8>> {
-        let yuvs = yuvs;
-        let mut bgra = vec![];
-        let mut v = vec![];
-        let (linesize, offset, length) =
-            ffmpeg_linesize_offset_length(pixfmt, width, height, HW_STRIDE_ALIGN).unwrap();
-        for mut yuv in yuvs {
-            i420_to_bgra(width, height, &yuv, &mut bgra);
-            if pixfmt == AVPixelFormat::AV_PIX_FMT_YUV420P {
-                hw_bgra_to_i420(width, height, &linesize, &offset, length, &bgra, &mut yuv);
-            } else {
-                hw_bgra_to_nv12(width, height, &linesize, &offset, length, &bgra, &mut yuv);
-            }
-            v.push(yuv);
-        }
-        v
+        println!("{} {}: {:?}", info.name, device, start.elapsed() / cnt);
     }
 }

libs/scrap/examples/capture_mag.rs

@@ -3,7 +3,7 @@ extern crate scrap;
 
 use scrap::Display;
 #[cfg(windows)]
-use scrap::{i420_to_rgb, CapturerMag, TraitCapturer};
+use scrap::{CapturerMag, TraitCapturer};
 #[cfg(windows)]
 use std::fs::File;
 
@@ -24,6 +24,8 @@ fn get_display(i: usize) -> Display {
 fn record(i: usize) {
     use std::time::Duration;
 
+    use scrap::TraitFrame;
+
     for d in Display::all().unwrap() {
         println!("{:?} {} {}", d.origin(), d.width(), d.height());
     }
@@ -32,9 +34,8 @@ fn record(i: usize) {
     let (w, h) = (display.width(), display.height());
 
     {
-        let mut capture_mag =
-            CapturerMag::new(display.origin(), display.width(), display.height(), false)
-                .expect("Couldn't begin capture.");
+        let mut capture_mag = CapturerMag::new(display.origin(), display.width(), display.height())
+            .expect("Couldn't begin capture.");
         let wnd_cls = "";
         let wnd_name = "RustDeskPrivacyWindow";
         if false == capture_mag.exclude(wnd_cls, wnd_name).unwrap() {
@@ -43,7 +44,8 @@ fn record(i: usize) {
             println!("Filter window for cls {} name {}", wnd_cls, wnd_name);
         }
 
-        let frame = capture_mag.frame(Duration::from_millis(0)).unwrap();
+        let captured_frame = capture_mag.frame(Duration::from_millis(0)).unwrap();
+        let frame = captured_frame.data();
         println!("Capture data len: {}, Saving...", frame.len());
 
         let mut bitflipped = Vec::with_capacity(w * h * 4);
@@ -68,9 +70,8 @@ fn record(i: usize) {
     }
 
     {
-        let mut capture_mag =
-            CapturerMag::new(display.origin(), display.width(), display.height(), true)
-                .expect("Couldn't begin capture.");
+        let mut capture_mag = CapturerMag::new(display.origin(), display.width(), display.height())
+            .expect("Couldn't begin capture.");
         let wnd_cls = "";
         let wnd_title = "RustDeskPrivacyWindow";
         if false == capture_mag.exclude(wnd_cls, wnd_title).unwrap() {
@@ -79,19 +80,28 @@ fn record(i: usize) {
             println!("Filter window for cls {} title {}", wnd_cls, wnd_title);
         }
 
-        let buffer = capture_mag.frame(Duration::from_millis(0)).unwrap();
-        println!("Capture data len: {}, Saving...", buffer.len());
+        let frame = capture_mag.frame(Duration::from_millis(0)).unwrap();
+        println!("Capture data len: {}, Saving...", frame.data().len());
 
-        let mut frame = Default::default();
-        i420_to_rgb(w, h, &buffer, &mut frame);
+        let mut raw = Vec::new();
+        unsafe {
+            scrap::ARGBToRAW(
+                frame.data().as_ptr(),
+                frame.stride()[0] as _,
+                (&mut raw).as_mut_ptr(),
+                (w * 3) as _,
+                w as _,
+                h as _,
+            )
+        };
 
         let mut bitflipped = Vec::with_capacity(w * h * 4);
-        let stride = frame.len() / h;
+        let stride = raw.len() / h;
 
         for y in 0..h {
             for x in 0..w {
                 let i = stride * y + 3 * x;
-                bitflipped.extend_from_slice(&[frame[i], frame[i + 1], frame[i + 2], 255]);
+                bitflipped.extend_from_slice(&[raw[i], raw[i + 1], raw[i + 2], 255]);
             }
         }
         let name = format!("capture_mag_{}_2.png", i);

libs/scrap/examples/ffplay.rs

@@ -1,5 +1,7 @@
 use std::time::Duration;
 
+use scrap::TraitFrame;
+
 extern crate scrap;
 
 fn main() {
@@ -27,16 +29,16 @@ fn main() {
         .spawn()
         .expect("This example requires ffplay.");
 
-    let mut capturer = Capturer::new(d, false).unwrap();
+    let mut capturer = Capturer::new(d).unwrap();
     let mut out = child.stdin.unwrap();
 
     loop {
         match capturer.frame(Duration::from_millis(0)) {
             Ok(frame) => {
                 // Write the frame, removing end-of-row padding.
-                let stride = frame.len() / h;
+                let stride = frame.stride()[0];
                 let rowlen = 4 * w;
-                for row in frame.chunks(stride) {
+                for row in frame.data().chunks(stride) {
                     let row = &row[..rowlen];
                     out.write_all(row).unwrap();
                 }

libs/scrap/examples/record-screen.rs

@@ -17,7 +17,7 @@ use scrap::codec::{EncoderApi, EncoderCfg, Quality as Q};
 use webm::mux;
 use webm::mux::Track;
 
-use scrap::vpxcodec as vpx_encode;
+use scrap::{convert_to_yuv, vpxcodec as vpx_encode};
 use scrap::{Capturer, Display, TraitCapturer, STRIDE_ALIGN};
 
 const USAGE: &'static str = "
@@ -110,13 +110,16 @@ fn main() -> io::Result<()> {
         Quality::Balanced => Q::Balanced,
         Quality::Low => Q::Low,
     };
-    let mut vpx = vpx_encode::VpxEncoder::new(EncoderCfg::VPX(vpx_encode::VpxEncoderConfig {
-        width,
-        height,
-        quality,
-        codec: vpx_codec,
-        keyframe_interval: None,
-    }))
+    let mut vpx = vpx_encode::VpxEncoder::new(
+        EncoderCfg::VPX(vpx_encode::VpxEncoderConfig {
+            width,
+            height,
+            quality,
+            codec: vpx_codec,
+            keyframe_interval: None,
+        }),
+        false,
+    )
     .unwrap();
 
     // Start recording.
@@ -136,7 +139,9 @@ fn main() -> io::Result<()> {
     let spf = Duration::from_nanos(1_000_000_000 / args.flag_fps);
 
     // Capturer object is expensive, avoiding to create it frequently.
-    let mut c = Capturer::new(d, true).unwrap();
+    let mut c = Capturer::new(d).unwrap();
+    let mut yuv = Vec::new();
+    let mut mid_data = Vec::new();
     while !stop.load(Ordering::Acquire) {
         let now = Instant::now();
         let time = now - start;
@@ -147,8 +152,8 @@ fn main() -> io::Result<()> {
 
         if let Ok(frame) = c.frame(Duration::from_millis(0)) {
             let ms = time.as_secs() * 1000 + time.subsec_millis() as u64;
-
-            for frame in vpx.encode(ms as i64, &frame, STRIDE_ALIGN).unwrap() {
+            convert_to_yuv(&frame, vpx.yuvfmt(), &mut yuv, &mut mid_data);
+            for frame in vpx.encode(ms as i64, &yuv, STRIDE_ALIGN).unwrap() {
                 vt.add_frame(frame.data, frame.pts as u64 * 1_000_000, frame.key);
             }
         }

libs/scrap/examples/screenshot.rs

@@ -6,7 +6,7 @@ use std::io::ErrorKind::WouldBlock;
 use std::thread;
 use std::time::Duration;
 
-use scrap::{i420_to_rgb, Capturer, Display, TraitCapturer};
+use scrap::{Capturer, Display, TraitCapturer, TraitFrame};
 
 fn main() {
     let n = Display::all().unwrap().len();
@@ -28,14 +28,14 @@ fn record(i: usize) {
     }
 
     let display = get_display(i);
-    let mut capturer = Capturer::new(display, false).expect("Couldn't begin capture.");
+    let mut capturer = Capturer::new(display).expect("Couldn't begin capture.");
     let (w, h) = (capturer.width(), capturer.height());
 
     loop {
         // Wait until there's a frame.
 
-        let buffer = match capturer.frame(Duration::from_millis(0)) {
-            Ok(buffer) => buffer,
+        let frame = match capturer.frame(Duration::from_millis(0)) {
+            Ok(frame) => frame,
             Err(error) => {
                 if error.kind() == WouldBlock {
                     // Keep spinning.
@@ -46,6 +46,7 @@ fn record(i: usize) {
                 }
             }
         };
+        let buffer = frame.data();
         println!("Captured data len: {}, Saving...", buffer.len());
 
         // Flip the BGRA image into a RGBA image.
@@ -77,14 +78,14 @@ fn record(i: usize) {
 
     drop(capturer);
     let display = get_display(i);
-    let mut capturer = Capturer::new(display, true).expect("Couldn't begin capture.");
+    let mut capturer = Capturer::new(display).expect("Couldn't begin capture.");
     let (w, h) = (capturer.width(), capturer.height());
 
     loop {
         // Wait until there's a frame.
 
-        let buffer = match capturer.frame(Duration::from_millis(0)) {
-            Ok(buffer) => buffer,
+        let frame = match capturer.frame(Duration::from_millis(0)) {
+            Ok(frame) => frame,
             Err(error) => {
                 if error.kind() == WouldBlock {
                     // Keep spinning.
@@ -95,18 +96,28 @@ fn record(i: usize) {
                 }
             }
         };
+        let buffer = frame.data();
         println!("Captured data len: {}, Saving...", buffer.len());
 
-        let mut frame = Default::default();
-        i420_to_rgb(w, h, &buffer, &mut frame);
+        let mut raw = Vec::new();
+        unsafe {
+            scrap::ARGBToRAW(
+                buffer.as_ptr(),
+                frame.stride()[0] as _,
+                (&mut raw).as_mut_ptr(),
+                (w * 3) as _,
+                w as _,
+                h as _,
+            )
+        };
 
         let mut bitflipped = Vec::with_capacity(w * h * 4);
-        let stride = frame.len() / h;
+        let stride = raw.len() / h;
 
         for y in 0..h {
             for x in 0..w {
                 let i = stride * y + 3 * x;
-                bitflipped.extend_from_slice(&[frame[i], frame[i + 1], frame[i + 2], 255]);
+                bitflipped.extend_from_slice(&[raw[i], raw[i + 1], raw[i + 2], 255]);
             }
         }
         let name = format!("screenshot{}_2.png", i);

libs/scrap/src/bindings/yuv_ffi.h

@@ -0,0 +1,6 @@
+#include <libyuv/convert.h>
+#include <libyuv/convert_argb.h>
+#include <libyuv/convert_from.h>
+#include <libyuv/convert_from_argb.h>
+#include <libyuv/rotate.h>
+#include <libyuv/rotate_argb.h>

libs/scrap/src/common/android.rs

@@ -1,5 +1,5 @@
 use crate::android::ffi::*;
-use crate::rgba_to_i420;
+use crate::Pixfmt;
 use lazy_static::lazy_static;
 use serde_json::Value;
 use std::collections::HashMap;
@@ -12,15 +12,15 @@ lazy_static! {
 
 pub struct Capturer {
     display: Display,
-    bgra: Vec<u8>,
+    rgba: Vec<u8>,
     saved_raw_data: Vec<u8>, // for faster compare and copy
 }
 
 impl Capturer {
-    pub fn new(display: Display, _yuv: bool) -> io::Result<Capturer> {
+    pub fn new(display: Display) -> io::Result<Capturer> {
         Ok(Capturer {
             display,
-            bgra: Vec::new(),
+            rgba: Vec::new(),
             saved_raw_data: Vec::new(),
         })
     }
@@ -35,22 +35,62 @@ impl Capturer {
 }
 
 impl crate::TraitCapturer for Capturer {
-    fn set_use_yuv(&mut self, _use_yuv: bool) {}
-
     fn frame<'a>(&'a mut self, _timeout: Duration) -> io::Result<Frame<'a>> {
         if let Some(buf) = get_video_raw() {
             crate::would_block_if_equal(&mut self.saved_raw_data, buf)?;
-            rgba_to_i420(self.width(), self.height(), buf, &mut self.bgra);
-            Ok(Frame::RAW(&self.bgra))
+            // Is it safe to directly return buf without copy?
+            self.rgba.resize(buf.len(), 0);
+            unsafe {
+                std::ptr::copy_nonoverlapping(buf.as_ptr(), self.rgba.as_mut_ptr(), buf.len())
+            };
+            Ok(Frame::new(&self.rgba, self.width(), self.height()))
         } else {
             return Err(io::ErrorKind::WouldBlock.into());
         }
     }
 }
 
-pub enum Frame<'a> {
-    RAW(&'a [u8]),
-    Empty,
+pub struct Frame<'a> {
+    data: &'a [u8],
+    width: usize,
+    height: usize,
+    stride: Vec<usize>,
+}
+
+impl<'a> Frame<'a> {
+    pub fn new(data: &'a [u8], width: usize, height: usize) -> Self {
+        let stride0 = data.len() / height;
+        let mut stride = Vec::new();
+        stride.push(stride0);
+        Frame {
+            data,
+            width,
+            height,
+            stride,
+        }
+    }
+}
+
+impl<'a> crate::TraitFrame for Frame<'a> {
+    fn data(&self) -> &[u8] {
+        self.data
+    }
+
+    fn width(&self) -> usize {
+        self.width
+    }
+
+    fn height(&self) -> usize {
+        self.height
+    }
+
+    fn stride(&self) -> Vec<usize> {
+        self.stride.clone()
+    }
+
+    fn pixfmt(&self) -> Pixfmt {
+        Pixfmt::RGBA
+    }
 }
 
 pub struct Display {

libs/scrap/src/common/aom.rs

@@ -9,11 +9,12 @@ include!(concat!(env!("OUT_DIR"), "/aom_ffi.rs"));
 use crate::codec::{base_bitrate, codec_thread_num, Quality};
 use crate::{codec::EncoderApi, EncodeFrame, STRIDE_ALIGN};
 use crate::{common::GoogleImage, generate_call_macro, generate_call_ptr_macro, Error, Result};
+use crate::{EncodeYuvFormat, Pixfmt};
 use hbb_common::{
     anyhow::{anyhow, Context},
     bytes::Bytes,
     log,
-    message_proto::{EncodedVideoFrame, EncodedVideoFrames, VideoFrame},
+    message_proto::{Chroma, EncodedVideoFrame, EncodedVideoFrames, VideoFrame},
     ResultType,
 };
 use std::{ptr, slice};
@@ -52,6 +53,8 @@ pub struct AomEncoder {
     ctx: aom_codec_ctx_t,
     width: usize,
     height: usize,
+    i444: bool,
+    yuvfmt: EncodeYuvFormat,
 }
 
 // https://webrtc.googlesource.com/src/+/refs/heads/main/modules/video_coding/codecs/av1/libaom_av1_encoder.cc
@@ -95,6 +98,7 @@ mod webrtc {
     pub fn enc_cfg(
         i: *const aom_codec_iface,
         cfg: AomEncoderConfig,
+        i444: bool,
     ) -> ResultType<aom_codec_enc_cfg> {
         let mut c = unsafe { std::mem::MaybeUninit::zeroed().assume_init() };
         call_aom!(aom_codec_enc_config_default(i, &mut c, kUsageProfile));
@@ -139,6 +143,9 @@ mod webrtc {
         c.g_pass = aom_enc_pass::AOM_RC_ONE_PASS; // One-pass rate control
         c.g_lag_in_frames = kLagInFrames; // No look ahead when lag equals 0.
 
+        // https://aomedia.googlesource.com/aom/+/refs/tags/v3.6.0/av1/common/enums.h#82
+        c.g_profile = if i444 { 1 } else { 0 };
+
         Ok(c)
     }
 
@@ -210,14 +217,14 @@ mod webrtc {
 }
 
 impl EncoderApi for AomEncoder {
-    fn new(cfg: crate::codec::EncoderCfg) -> ResultType<Self>
+    fn new(cfg: crate::codec::EncoderCfg, i444: bool) -> ResultType<Self>
     where
         Self: Sized,
     {
         match cfg {
             crate::codec::EncoderCfg::AOM(config) => {
                 let i = call_aom_ptr!(aom_codec_av1_cx());
-                let c = webrtc::enc_cfg(i, config)?;
+                let c = webrtc::enc_cfg(i, config, i444)?;
 
                 let mut ctx = Default::default();
                 // Flag options: AOM_CODEC_USE_PSNR and AOM_CODEC_USE_HIGHBITDEPTH
@@ -234,6 +241,8 @@ impl EncoderApi for AomEncoder {
                     ctx,
                     width: config.width as _,
                     height: config.height as _,
+                    i444,
+                    yuvfmt: Self::get_yuvfmt(config.width, config.height, i444),
                 })
             }
             _ => Err(anyhow!("encoder type mismatch")),
@@ -255,8 +264,8 @@ impl EncoderApi for AomEncoder {
         }
     }
 
-    fn use_yuv(&self) -> bool {
-        true
+    fn yuvfmt(&self) -> crate::EncodeYuvFormat {
+        self.yuvfmt.clone()
     }
 
     fn set_quality(&mut self, quality: Quality) -> ResultType<()> {
@@ -282,14 +291,20 @@ impl EncoderApi for AomEncoder {
 
 impl AomEncoder {
     pub fn encode(&mut self, pts: i64, data: &[u8], stride_align: usize) -> Result<EncodeFrames> {
-        if 2 * data.len() < 3 * self.width * self.height {
+        let bpp = if self.i444 { 24 } else { 12 };
+        if data.len() < self.width * self.height * bpp / 8 {
             return Err(Error::FailedCall("len not enough".to_string()));
         }
+        let fmt = if self.i444 {
+            aom_img_fmt::AOM_IMG_FMT_I444
+        } else {
+            aom_img_fmt::AOM_IMG_FMT_I420
+        };
 
         let mut image = Default::default();
         call_aom_ptr!(aom_img_wrap(
             &mut image,
-            aom_img_fmt::AOM_IMG_FMT_I420,
+            fmt,
             self.width as _,
             self.height as _,
             stride_align as _,
@@ -359,6 +374,34 @@ impl AomEncoder {
 
         (q_min, q_max)
     }
+
+    fn get_yuvfmt(width: u32, height: u32, i444: bool) -> EncodeYuvFormat {
+        let mut img = Default::default();
+        let fmt = if i444 {
+            aom_img_fmt::AOM_IMG_FMT_I444
+        } else {
+            aom_img_fmt::AOM_IMG_FMT_I420
+        };
+        unsafe {
+            aom_img_wrap(
+                &mut img,
+                fmt,
+                width as _,
+                height as _,
+                crate::STRIDE_ALIGN as _,
+                0x1 as _,
+            );
+        }
+        let pixfmt = if i444 { Pixfmt::I444 } else { Pixfmt::I420 };
+        EncodeYuvFormat {
+            pixfmt,
+            w: img.w as _,
+            h: img.h as _,
+            stride: img.stride.map(|s| s as usize).to_vec(),
+            u: img.planes[1] as usize - img.planes[0] as usize,
+            v: img.planes[2] as usize - img.planes[0] as usize,
+        }
+    }
 }
 
 impl Drop for AomEncoder {
@@ -524,6 +567,13 @@ impl GoogleImage for Image {
     fn planes(&self) -> Vec<*mut u8> {
         self.inner().planes.iter().map(|p| *p as *mut u8).collect()
     }
+
+    fn chroma(&self) -> Chroma {
+        match self.inner().fmt {
+            aom_img_fmt::AOM_IMG_FMT_I444 => Chroma::I444,
+            _ => Chroma::I420,
+        }
+    }
 }
 
 impl Drop for Image {

libs/scrap/src/common/codec.rs

@@ -14,7 +14,7 @@ use crate::{
     aom::{self, AomDecoder, AomEncoder, AomEncoderConfig},
     common::GoogleImage,
     vpxcodec::{self, VpxDecoder, VpxDecoderConfig, VpxEncoder, VpxEncoderConfig, VpxVideoCodecId},
-    CodecName, ImageRgb,
+    CodecName, EncodeYuvFormat, ImageRgb,
 };
 
 use hbb_common::{
@@ -23,7 +23,7 @@ use hbb_common::{
     config::PeerConfig,
     log,
     message_proto::{
-        supported_decoding::PreferCodec, video_frame, EncodedVideoFrames,
+        supported_decoding::PreferCodec, video_frame, Chroma, CodecAbility, EncodedVideoFrames,
         SupportedDecoding, SupportedEncoding, VideoFrame,
     },
     sysinfo::{System, SystemExt},
@@ -56,13 +56,13 @@ pub enum EncoderCfg {
 }
 
 pub trait EncoderApi {
-    fn new(cfg: EncoderCfg) -> ResultType<Self>
+    fn new(cfg: EncoderCfg, i444: bool) -> ResultType<Self>
     where
         Self: Sized;
 
     fn encode_to_message(&mut self, frame: &[u8], ms: i64) -> ResultType<VideoFrame>;
 
-    fn use_yuv(&self) -> bool;
+    fn yuvfmt(&self) -> EncodeYuvFormat;
 
     fn set_quality(&mut self, quality: Quality) -> ResultType<()>;
 
@@ -107,18 +107,18 @@ pub enum EncodingUpdate {
 }
 
 impl Encoder {
-    pub fn new(config: EncoderCfg) -> ResultType<Encoder> {
-        log::info!("new encoder:{:?}", config);
+    pub fn new(config: EncoderCfg, i444: bool) -> ResultType<Encoder> {
+        log::info!("new encoder:{config:?}, i444:{i444}");
         match config {
             EncoderCfg::VPX(_) => Ok(Encoder {
-                codec: Box::new(VpxEncoder::new(config)?),
+                codec: Box::new(VpxEncoder::new(config, i444)?),
             }),
             EncoderCfg::AOM(_) => Ok(Encoder {
-                codec: Box::new(AomEncoder::new(config)?),
+                codec: Box::new(AomEncoder::new(config, i444)?),
             }),
 
             #[cfg(feature = "hwcodec")]
-            EncoderCfg::HW(_) => match HwEncoder::new(config) {
+            EncoderCfg::HW(_) => match HwEncoder::new(config, i444) {
                 Ok(hw) => Ok(Encoder {
                     codec: Box::new(hw),
                 }),
@@ -230,6 +230,12 @@ impl Encoder {
         let mut encoding = SupportedEncoding {
             vp8: true,
             av1: true,
+            i444: Some(CodecAbility {
+                vp9: true,
+                av1: true,
+                ..Default::default()
+            })
+            .into(),
             ..Default::default()
         };
         #[cfg(feature = "hwcodec")]
@@ -240,18 +246,41 @@ impl Encoder {
         }
         encoding
     }
+
+    pub fn use_i444(config: &EncoderCfg) -> bool {
+        let decodings = PEER_DECODINGS.lock().unwrap().clone();
+        let prefer_i444 = decodings
+            .iter()
+            .all(|d| d.1.prefer_chroma == Chroma::I444.into());
+        let i444_useable = match config {
+            EncoderCfg::VPX(vpx) => match vpx.codec {
+                VpxVideoCodecId::VP8 => false,
+                VpxVideoCodecId::VP9 => decodings.iter().all(|d| d.1.i444.vp9),
+            },
+            EncoderCfg::AOM(_) => decodings.iter().all(|d| d.1.i444.av1),
+            EncoderCfg::HW(_) => false,
+        };
+        prefer_i444 && i444_useable && !decodings.is_empty()
+    }
 }
 
 impl Decoder {
     pub fn supported_decodings(id_for_perfer: Option<&str>) -> SupportedDecoding {
+        let (prefer, prefer_chroma) = Self::preference(id_for_perfer);
+
         #[allow(unused_mut)]
         let mut decoding = SupportedDecoding {
             ability_vp8: 1,
             ability_vp9: 1,
             ability_av1: 1,
-            prefer: id_for_perfer
-                .map_or(PreferCodec::Auto, |id| Self::codec_preference(id))
-                .into(),
+            i444: Some(CodecAbility {
+                vp9: true,
+                av1: true,
+                ..Default::default()
+            })
+            .into(),
+            prefer: prefer.into(),
+            prefer_chroma: prefer_chroma.into(),
             ..Default::default()
         };
         #[cfg(feature = "hwcodec")]
@@ -314,31 +343,33 @@ impl Decoder {
         &mut self,
         frame: &video_frame::Union,
         rgb: &mut ImageRgb,
+        chroma: &mut Option<Chroma>,
     ) -> ResultType<bool> {
         match frame {
             video_frame::Union::Vp8s(vp8s) => {
                 if let Some(vp8) = &mut self.vp8 {
-                    Decoder::handle_vpxs_video_frame(vp8, vp8s, rgb)
+                    Decoder::handle_vpxs_video_frame(vp8, vp8s, rgb, chroma)
                 } else {
                     bail!("vp8 decoder not available");
                 }
             }
             video_frame::Union::Vp9s(vp9s) => {
                 if let Some(vp9) = &mut self.vp9 {
-                    Decoder::handle_vpxs_video_frame(vp9, vp9s, rgb)
+                    Decoder::handle_vpxs_video_frame(vp9, vp9s, rgb, chroma)
                 } else {
                     bail!("vp9 decoder not available");
                 }
             }
             video_frame::Union::Av1s(av1s) => {
                 if let Some(av1) = &mut self.av1 {
-                    Decoder::handle_av1s_video_frame(av1, av1s, rgb)
+                    Decoder::handle_av1s_video_frame(av1, av1s, rgb, chroma)
                 } else {
                     bail!("av1 decoder not available");
                 }
             }
             #[cfg(feature = "hwcodec")]
             video_frame::Union::H264s(h264s) => {
+                *chroma = Some(Chroma::I420);
                 if let Some(decoder) = &mut self.hw.h264 {
                     Decoder::handle_hw_video_frame(decoder, h264s, rgb, &mut self.i420)
                 } else {
@@ -347,6 +378,7 @@ impl Decoder {
             }
             #[cfg(feature = "hwcodec")]
             video_frame::Union::H265s(h265s) => {
+                *chroma = Some(Chroma::I420);
                 if let Some(decoder) = &mut self.hw.h265 {
                     Decoder::handle_hw_video_frame(decoder, h265s, rgb, &mut self.i420)
                 } else {
@@ -355,6 +387,7 @@ impl Decoder {
             }
             #[cfg(feature = "mediacodec")]
             video_frame::Union::H264s(h264s) => {
+                *chroma = Some(Chroma::I420);
                 if let Some(decoder) = &mut self.media_codec.h264 {
                     Decoder::handle_mediacodec_video_frame(decoder, h264s, rgb)
                 } else {
@@ -363,6 +396,7 @@ impl Decoder {
             }
             #[cfg(feature = "mediacodec")]
             video_frame::Union::H265s(h265s) => {
+                *chroma = Some(Chroma::I420);
                 if let Some(decoder) = &mut self.media_codec.h265 {
                     Decoder::handle_mediacodec_video_frame(decoder, h265s, rgb)
                 } else {
@@ -378,6 +412,7 @@ impl Decoder {
         decoder: &mut VpxDecoder,
         vpxs: &EncodedVideoFrames,
         rgb: &mut ImageRgb,
+        chroma: &mut Option<Chroma>,
     ) -> ResultType<bool> {
         let mut last_frame = vpxcodec::Image::new();
         for vpx in vpxs.frames.iter() {
@@ -393,6 +428,7 @@ impl Decoder {
         if last_frame.is_null() {
             Ok(false)
         } else {
+            *chroma = Some(last_frame.chroma());
             last_frame.to(rgb);
             Ok(true)
         }
@@ -403,6 +439,7 @@ impl Decoder {
         decoder: &mut AomDecoder,
         av1s: &EncodedVideoFrames,
         rgb: &mut ImageRgb,
+        chroma: &mut Option<Chroma>,
     ) -> ResultType<bool> {
         let mut last_frame = aom::Image::new();
         for av1 in av1s.frames.iter() {
@@ -418,6 +455,7 @@ impl Decoder {
         if last_frame.is_null() {
             Ok(false)
         } else {
+            *chroma = Some(last_frame.chroma());
             last_frame.to(rgb);
             Ok(true)
         }
@@ -457,12 +495,16 @@ impl Decoder {
         return Ok(false);
     }
 
-    fn codec_preference(id: &str) -> PreferCodec {
-        let codec = PeerConfig::load(id)
-            .options
+    fn preference(id: Option<&str>) -> (PreferCodec, Chroma) {
+        let id = id.unwrap_or_default();
+        if id.is_empty() {
+            return (PreferCodec::Auto, Chroma::I420);
+        }
+        let options = PeerConfig::load(id).options;
+        let codec = options
             .get("codec-preference")
             .map_or("".to_owned(), |c| c.to_owned());
-        if codec == "vp8" {
+        let codec = if codec == "vp8" {
             PreferCodec::VP8
         } else if codec == "vp9" {
             PreferCodec::VP9
@@ -474,7 +516,13 @@ impl Decoder {
             PreferCodec::H265
         } else {
             PreferCodec::Auto
-        }
+        };
+        let chroma = if options.get("i444") == Some(&"Y".to_string()) {
+            Chroma::I444
+        } else {
+            Chroma::I420
+        };
+        (codec, chroma)
     }
 }
 

libs/scrap/src/common/convert.rs

@@ -1,367 +1,25 @@
-use super::vpx::*;
-use std::os::raw::c_int;
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+#![allow(non_upper_case_globals)]
+#![allow(improper_ctypes)]
+#![allow(dead_code)]
 
-extern "C" {
-    // seems libyuv uses reverse byte order compared with our view
+include!(concat!(env!("OUT_DIR"), "/yuv_ffi.rs"));
 
-    pub fn ARGBRotate(
-        src_argb: *const u8,
-        src_stride_argb: c_int,
-        dst_argb: *mut u8,
-        dst_stride_argb: c_int,
-        src_width: c_int,
-        src_height: c_int,
-        mode: c_int,
-    ) -> c_int;
+#[cfg(not(target_os = "ios"))]
+use crate::Frame;
+use crate::{generate_call_macro, EncodeYuvFormat, TraitFrame};
+use hbb_common::{bail, log, ResultType};
 
-    pub fn ARGBMirror(
-        src_argb: *const u8,
-        src_stride_argb: c_int,
-        dst_argb: *mut u8,
-        dst_stride_argb: c_int,
-        width: c_int,
-        height: c_int,
-    ) -> c_int;
-
-    pub fn ARGBToI420(
-        src_bgra: *const u8,
-        src_stride_bgra: c_int,
-        dst_y: *mut u8,
-        dst_stride_y: c_int,
-        dst_u: *mut u8,
-        dst_stride_u: c_int,
-        dst_v: *mut u8,
-        dst_stride_v: c_int,
-        width: c_int,
-        height: c_int,
-    ) -> c_int;
-
-    pub fn ABGRToI420(
-        src_rgba: *const u8,
-        src_stride_rgba: c_int,
-        dst_y: *mut u8,
-        dst_stride_y: c_int,
-        dst_u: *mut u8,
-        dst_stride_u: c_int,
-        dst_v: *mut u8,
-        dst_stride_v: c_int,
-        width: c_int,
-        height: c_int,
-    ) -> c_int;
-
-    pub fn ARGBToNV12(
-        src_bgra: *const u8,
-        src_stride_bgra: c_int,
-        dst_y: *mut u8,
-        dst_stride_y: c_int,
-        dst_uv: *mut u8,
-        dst_stride_uv: c_int,
-        width: c_int,
-        height: c_int,
-    ) -> c_int;
-
-    pub fn NV12ToI420(
-        src_y: *const u8,
-        src_stride_y: c_int,
-        src_uv: *const u8,
-        src_stride_uv: c_int,
-        dst_y: *mut u8,
-        dst_stride_y: c_int,
-        dst_u: *mut u8,
-        dst_stride_u: c_int,
-        dst_v: *mut u8,
-        dst_stride_v: c_int,
-        width: c_int,
-        height: c_int,
-    ) -> c_int;
-
-    // I420ToRGB24: RGB little endian (bgr in memory)
-    // I420ToRaw: RGB big endian (rgb in memory) to RGBA.
-    pub fn I420ToRAW(
-        src_y: *const u8,
-        src_stride_y: c_int,
-        src_u: *const u8,
-        src_stride_u: c_int,
-        src_v: *const u8,
-        src_stride_v: c_int,
-        dst_rgba: *mut u8,
-        dst_stride_raw: c_int,
-        width: c_int,
-        height: c_int,
-    ) -> c_int;
-
-    pub fn I420ToARGB(
-        src_y: *const u8,
-        src_stride_y: c_int,
-        src_u: *const u8,
-        src_stride_u: c_int,
-        src_v: *const u8,
-        src_stride_v: c_int,
-        dst_rgba: *mut u8,
-        dst_stride_rgba: c_int,
-        width: c_int,
-        height: c_int,
-    ) -> c_int;
-
-    pub fn I420ToABGR(
-        src_y: *const u8,
-        src_stride_y: c_int,
-        src_u: *const u8,
-        src_stride_u: c_int,
-        src_v: *const u8,
-        src_stride_v: c_int,
-        dst_rgba: *mut u8,
-        dst_stride_rgba: c_int,
-        width: c_int,
-        height: c_int,
-    ) -> c_int;
-
-    pub fn NV12ToARGB(
-        src_y: *const u8,
-        src_stride_y: c_int,
-        src_uv: *const u8,
-        src_stride_uv: c_int,
-        dst_rgba: *mut u8,
-        dst_stride_rgba: c_int,
-        width: c_int,
-        height: c_int,
-    ) -> c_int;
-
-    pub fn NV12ToABGR(
-        src_y: *const u8,
-        src_stride_y: c_int,
-        src_uv: *const u8,
-        src_stride_uv: c_int,
-        dst_rgba: *mut u8,
-        dst_stride_rgba: c_int,
-        width: c_int,
-        height: c_int,
-    ) -> c_int;
-}
-
-// https://github.com/webmproject/libvpx/blob/master/vpx/src/vpx_image.c
-#[inline]
-fn get_vpx_i420_stride(
-    width: usize,
-    height: usize,
-    stride_align: usize,
-) -> (usize, usize, usize, usize, usize, usize) {
-    let mut img = Default::default();
-    unsafe {
-        vpx_img_wrap(
-            &mut img,
-            vpx_img_fmt::VPX_IMG_FMT_I420,
-            width as _,
-            height as _,
-            stride_align as _,
-            0x1 as _,
-        );
-    }
-    (
-        img.w as _,
-        img.h as _,
-        img.stride[0] as _,
-        img.stride[1] as _,
-        img.planes[1] as usize - img.planes[0] as usize,
-        img.planes[2] as usize - img.planes[0] as usize,
-    )
-}
-
-pub fn i420_to_rgb(width: usize, height: usize, src: &[u8], dst: &mut Vec<u8>) {
-    let (_, _, src_stride_y, src_stride_uv, u, v) =
-        get_vpx_i420_stride(width, height, super::STRIDE_ALIGN);
-    let src_y = src.as_ptr();
-    let src_u = src[u..].as_ptr();
-    let src_v = src[v..].as_ptr();
-    dst.resize(width * height * 3, 0);
-    unsafe {
-        super::I420ToRAW(
-            src_y,
-            src_stride_y as _,
-            src_u,
-            src_stride_uv as _,
-            src_v,
-            src_stride_uv as _,
-            dst.as_mut_ptr(),
-            (width * 3) as _,
-            width as _,
-            height as _,
-        );
-    };
-}
-
-pub fn i420_to_bgra(width: usize, height: usize, src: &[u8], dst: &mut Vec<u8>) {
-    let (_, _, src_stride_y, src_stride_uv, u, v) =
-        get_vpx_i420_stride(width, height, super::STRIDE_ALIGN);
-    let src_y = src.as_ptr();
-    let src_u = src[u..].as_ptr();
-    let src_v = src[v..].as_ptr();
-    dst.resize(width * height * 4, 0);
-    unsafe {
-        super::I420ToARGB(
-            src_y,
-            src_stride_y as _,
-            src_u,
-            src_stride_uv as _,
-            src_v,
-            src_stride_uv as _,
-            dst.as_mut_ptr(),
-            (width * 3) as _,
-            width as _,
-            height as _,
-        );
-    };
-}
-
-pub fn bgra_to_i420(width: usize, height: usize, src: &[u8], dst: &mut Vec<u8>) {
-    let (_, h, dst_stride_y, dst_stride_uv, u, v) =
-        get_vpx_i420_stride(width, height, super::STRIDE_ALIGN);
-    dst.resize(h * dst_stride_y * 2, 0); // waste some memory to ensure memory safety
-    let dst_y = dst.as_mut_ptr();
-    let dst_u = dst[u..].as_mut_ptr();
-    let dst_v = dst[v..].as_mut_ptr();
-    unsafe {
-        ARGBToI420(
-            src.as_ptr(),
-            (src.len() / height) as _,
-            dst_y,
-            dst_stride_y as _,
-            dst_u,
-            dst_stride_uv as _,
-            dst_v,
-            dst_stride_uv as _,
-            width as _,
-            height as _,
-        );
-    }
-}
-
-pub fn rgba_to_i420(width: usize, height: usize, src: &[u8], dst: &mut Vec<u8>) {
-    let (_, h, dst_stride_y, dst_stride_uv, u, v) =
-        get_vpx_i420_stride(width, height, super::STRIDE_ALIGN);
-    dst.resize(h * dst_stride_y * 2, 0); // waste some memory to ensure memory safety
-    let dst_y = dst.as_mut_ptr();
-    let dst_u = dst[u..].as_mut_ptr();
-    let dst_v = dst[v..].as_mut_ptr();
-    unsafe {
-        ABGRToI420(
-            src.as_ptr(),
-            (src.len() / height) as _,
-            dst_y,
-            dst_stride_y as _,
-            dst_u,
-            dst_stride_uv as _,
-            dst_v,
-            dst_stride_uv as _,
-            width as _,
-            height as _,
-        );
-    }
-}
-
-pub unsafe fn nv12_to_i420(
-    src_y: *const u8,
-    src_stride_y: c_int,
-    src_uv: *const u8,
-    src_stride_uv: c_int,
-    width: usize,
-    height: usize,
-    dst: &mut Vec<u8>,
-) {
-    let (_, h, dst_stride_y, dst_stride_uv, u, v) =
-        get_vpx_i420_stride(width, height, super::STRIDE_ALIGN);
-    dst.resize(h * dst_stride_y * 2, 0); // waste some memory to ensure memory safety
-    let dst_y = dst.as_mut_ptr();
-    let dst_u = dst[u..].as_mut_ptr();
-    let dst_v = dst[v..].as_mut_ptr();
-    NV12ToI420(
-        src_y,
-        src_stride_y,
-        src_uv,
-        src_stride_uv,
-        dst_y,
-        dst_stride_y as _,
-        dst_u,
-        dst_stride_uv as _,
-        dst_v,
-        dst_stride_uv as _,
-        width as _,
-        height as _,
-    );
-}
+generate_call_macro!(call_yuv, false);
 
 #[cfg(feature = "hwcodec")]
 pub mod hw {
+    use super::*;
     use crate::ImageFormat;
-    use hbb_common::{anyhow::anyhow, ResultType};
     #[cfg(target_os = "windows")]
     use hwcodec::{ffmpeg::ffmpeg_linesize_offset_length, AVPixelFormat};
 
-    pub fn hw_bgra_to_i420(
-        width: usize,
-        height: usize,
-        stride: &[i32],
-        offset: &[i32],
-        length: i32,
-        src: &[u8],
-        dst: &mut Vec<u8>,
-    ) {
-        let stride_y = stride[0] as usize;
-        let stride_u = stride[1] as usize;
-        let stride_v = stride[2] as usize;
-        let offset_u = offset[0] as usize;
-        let offset_v = offset[1] as usize;
-
-        dst.resize(length as _, 0);
-        let dst_y = dst.as_mut_ptr();
-        let dst_u = dst[offset_u..].as_mut_ptr();
-        let dst_v = dst[offset_v..].as_mut_ptr();
-        unsafe {
-            super::ARGBToI420(
-                src.as_ptr(),
-                (src.len() / height) as _,
-                dst_y,
-                stride_y as _,
-                dst_u,
-                stride_u as _,
-                dst_v,
-                stride_v as _,
-                width as _,
-                height as _,
-            );
-        }
-    }
-
-    pub fn hw_bgra_to_nv12(
-        width: usize,
-        height: usize,
-        stride: &[i32],
-        offset: &[i32],
-        length: i32,
-        src: &[u8],
-        dst: &mut Vec<u8>,
-    ) {
-        let stride_y = stride[0] as usize;
-        let stride_uv = stride[1] as usize;
-        let offset_uv = offset[0] as usize;
-
-        dst.resize(length as _, 0);
-        let dst_y = dst.as_mut_ptr();
-        let dst_uv = dst[offset_uv..].as_mut_ptr();
-        unsafe {
-            super::ARGBToNV12(
-                src.as_ptr(),
-                (src.len() / height) as _,
-                dst_y,
-                stride_y as _,
-                dst_uv,
-                stride_uv as _,
-                width as _,
-                height as _,
-            );
-        }
-    }
-
     #[cfg(target_os = "windows")]
     pub fn hw_nv12_to(
         fmt: ImageFormat,
@@ -386,61 +44,59 @@ pub mod hw {
             let i420_stride_v = linesize_i420[2];
             i420.resize(i420_len as _, 0);
 
-            unsafe {
-                let i420_offset_y = i420.as_ptr().add(0) as _;
-                let i420_offset_u = i420.as_ptr().add(offset_i420[0] as _) as _;
-                let i420_offset_v = i420.as_ptr().add(offset_i420[1] as _) as _;
-                super::NV12ToI420(
-                    src_y.as_ptr(),
-                    nv12_stride_y as _,
-                    src_uv.as_ptr(),
-                    nv12_stride_uv as _,
-                    i420_offset_y,
-                    i420_stride_y,
-                    i420_offset_u,
-                    i420_stride_u,
-                    i420_offset_v,
-                    i420_stride_v,
-                    width as _,
-                    height as _,
-                );
-                match fmt {
-                    ImageFormat::ARGB => {
-                        super::I420ToARGB(
-                            i420_offset_y,
-                            i420_stride_y,
-                            i420_offset_u,
-                            i420_stride_u,
-                            i420_offset_v,
-                            i420_stride_v,
-                            dst.as_mut_ptr(),
-                            (width * 4) as _,
-                            width as _,
-                            height as _,
-                        );
-                    }
-                    ImageFormat::ABGR => {
-                        super::I420ToABGR(
-                            i420_offset_y,
-                            i420_stride_y,
-                            i420_offset_u,
-                            i420_stride_u,
-                            i420_offset_v,
-                            i420_stride_v,
-                            dst.as_mut_ptr(),
-                            (width * 4) as _,
-                            width as _,
-                            height as _,
-                        );
-                    }
-                    _ => {
-                        return Err(anyhow!("unsupported image format"));
-                    }
+            let i420_offset_y = unsafe { i420.as_ptr().add(0) as _ };
+            let i420_offset_u = unsafe { i420.as_ptr().add(offset_i420[0] as _) as _ };
+            let i420_offset_v = unsafe { i420.as_ptr().add(offset_i420[1] as _) as _ };
+            call_yuv!(NV12ToI420(
+                src_y.as_ptr(),
+                nv12_stride_y as _,
+                src_uv.as_ptr(),
+                nv12_stride_uv as _,
+                i420_offset_y,
+                i420_stride_y,
+                i420_offset_u,
+                i420_stride_u,
+                i420_offset_v,
+                i420_stride_v,
+                width as _,
+                height as _,
+            ));
+            match fmt {
+                ImageFormat::ARGB => {
+                    call_yuv!(I420ToARGB(
+                        i420_offset_y,
+                        i420_stride_y,
+                        i420_offset_u,
+                        i420_stride_u,
+                        i420_offset_v,
+                        i420_stride_v,
+                        dst.as_mut_ptr(),
+                        (width * 4) as _,
+                        width as _,
+                        height as _,
+                    ));
                 }
-                return Ok(());
-            };
+                ImageFormat::ABGR => {
+                    call_yuv!(I420ToABGR(
+                        i420_offset_y,
+                        i420_stride_y,
+                        i420_offset_u,
+                        i420_stride_u,
+                        i420_offset_v,
+                        i420_stride_v,
+                        dst.as_mut_ptr(),
+                        (width * 4) as _,
+                        width as _,
+                        height as _,
+                    ));
+                }
+                _ => {
+                    bail!("unsupported image format");
+                }
+            }
+            return Ok(());
         }
-        return Err(anyhow!("get linesize offset failed"));
+        bail!("get linesize offset failed");
     }
 
     #[cfg(not(target_os = "windows"))]
@@ -457,41 +113,34 @@ pub mod hw {
         _align: usize,
     ) -> ResultType<()> {
         dst.resize(width * height * 4, 0);
-        unsafe {
-            match fmt {
-                ImageFormat::ARGB => {
-                    match super::NV12ToARGB(
-                        src_y.as_ptr(),
-                        src_stride_y as _,
-                        src_uv.as_ptr(),
-                        src_stride_uv as _,
-                        dst.as_mut_ptr(),
-                        (width * 4) as _,
-                        width as _,
-                        height as _,
-                    ) {
-                        0 => Ok(()),
-                        _ => Err(anyhow!("NV12ToARGB failed")),
-                    }
-                }
-                ImageFormat::ABGR => {
-                    match super::NV12ToABGR(
-                        src_y.as_ptr(),
-                        src_stride_y as _,
-                        src_uv.as_ptr(),
-                        src_stride_uv as _,
-                        dst.as_mut_ptr(),
-                        (width * 4) as _,
-                        width as _,
-                        height as _,
-                    ) {
-                        0 => Ok(()),
-                        _ => Err(anyhow!("NV12ToABGR failed")),
-                    }
-                }
-                _ => Err(anyhow!("unsupported image format")),
+        match fmt {
+            ImageFormat::ARGB => {
+                call_yuv!(NV12ToARGB(
+                    src_y.as_ptr(),
+                    src_stride_y as _,
+                    src_uv.as_ptr(),
+                    src_stride_uv as _,
+                    dst.as_mut_ptr(),
+                    (width * 4) as _,
+                    width as _,
+                    height as _,
+                ));
             }
+            ImageFormat::ABGR => {
+                call_yuv!(NV12ToABGR(
+                    src_y.as_ptr(),
+                    src_stride_y as _,
+                    src_uv.as_ptr(),
+                    src_stride_uv as _,
+                    dst.as_mut_ptr(),
+                    (width * 4) as _,
+                    width as _,
+                    height as _,
+                ));
+            }
+            _ => bail!("unsupported image format"),
         }
+        Ok(())
     }
 
     pub fn hw_i420_to(
@@ -505,43 +154,173 @@ pub mod hw {
         src_stride_u: usize,
         src_stride_v: usize,
         dst: &mut Vec<u8>,
-    ) {
+    ) -> ResultType<()> {
         let src_y = src_y.as_ptr();
         let src_u = src_u.as_ptr();
         let src_v = src_v.as_ptr();
         dst.resize(width * height * 4, 0);
-        unsafe {
-            match fmt {
-                ImageFormat::ARGB => {
-                    super::I420ToARGB(
-                        src_y,
-                        src_stride_y as _,
-                        src_u,
-                        src_stride_u as _,
-                        src_v,
-                        src_stride_v as _,
-                        dst.as_mut_ptr(),
-                        (width * 4) as _,
-                        width as _,
-                        height as _,
-                    );
-                }
-                ImageFormat::ABGR => {
-                    super::I420ToABGR(
-                        src_y,
-                        src_stride_y as _,
-                        src_u,
-                        src_stride_u as _,
-                        src_v,
-                        src_stride_v as _,
-                        dst.as_mut_ptr(),
-                        (width * 4) as _,
-                        width as _,
-                        height as _,
-                    );
-                }
-                _ => {}
+        match fmt {
+            ImageFormat::ARGB => {
+                call_yuv!(I420ToARGB(
+                    src_y,
+                    src_stride_y as _,
+                    src_u,
+                    src_stride_u as _,
+                    src_v,
+                    src_stride_v as _,
+                    dst.as_mut_ptr(),
+                    (width * 4) as _,
+                    width as _,
+                    height as _,
+                ));
             }
+            ImageFormat::ABGR => {
+                call_yuv!(I420ToABGR(
+                    src_y,
+                    src_stride_y as _,
+                    src_u,
+                    src_stride_u as _,
+                    src_v,
+                    src_stride_v as _,
+                    dst.as_mut_ptr(),
+                    (width * 4) as _,
+                    width as _,
+                    height as _,
+                ));
+            }
+            _ => bail!("unsupported image format"),
         };
+        Ok(())
     }
 }
+#[cfg(not(target_os = "ios"))]
+pub fn convert_to_yuv(
+    captured: &Frame,
+    dst_fmt: EncodeYuvFormat,
+    dst: &mut Vec<u8>,
+    mid_data: &mut Vec<u8>,
+) -> ResultType<()> {
+    let src = captured.data();
+    let src_stride = captured.stride();
+    let src_pixfmt = captured.pixfmt();
+    let src_width = captured.width();
+    let src_height = captured.height();
+    if src_width > dst_fmt.w || src_height > dst_fmt.h {
+        bail!(
+            "src rect > dst rect: ({src_width}, {src_height}) > ({},{})",
+            dst_fmt.w,
+            dst_fmt.h
+        );
+    }
+    if src_pixfmt == crate::Pixfmt::BGRA || src_pixfmt == crate::Pixfmt::RGBA {
+        if src.len() < src_stride[0] * src_height {
+            bail!(
+                "wrong src len, {} < {} * {}",
+                src.len(),
+                src_stride[0],
+                src_height
+            );
+        }
+    }
+    let align = |x:usize| {
+        (x + 63) / 64 * 64
+    };
+
+    match (src_pixfmt, dst_fmt.pixfmt) {
+        (crate::Pixfmt::BGRA, crate::Pixfmt::I420) | (crate::Pixfmt::RGBA, crate::Pixfmt::I420) => {
+            let dst_stride_y = dst_fmt.stride[0];
+            let dst_stride_uv = dst_fmt.stride[1];
+            dst.resize(dst_fmt.h * dst_stride_y * 2, 0); // waste some memory to ensure memory safety
+            let dst_y = dst.as_mut_ptr();
+            let dst_u = dst[dst_fmt.u..].as_mut_ptr();
+            let dst_v = dst[dst_fmt.v..].as_mut_ptr();
+            let f = if src_pixfmt == crate::Pixfmt::BGRA {
+                ARGBToI420
+            } else {
+                ABGRToI420
+            };
+            call_yuv!(f(
+                src.as_ptr(),
+                src_stride[0] as _,
+                dst_y,
+                dst_stride_y as _,
+                dst_u,
+                dst_stride_uv as _,
+                dst_v,
+                dst_stride_uv as _,
+                src_width as _,
+                src_height as _,
+            ));
+        }
+        (crate::Pixfmt::BGRA, crate::Pixfmt::NV12) | (crate::Pixfmt::RGBA, crate::Pixfmt::NV12) => {
+            let dst_stride_y = dst_fmt.stride[0];
+            let dst_stride_uv = dst_fmt.stride[1];
+            dst.resize(
+                align(dst_fmt.h) * (align(dst_stride_y) + align(dst_stride_uv / 2)),
+                0,
+            );
+            let dst_y = dst.as_mut_ptr();
+            let dst_uv = dst[dst_fmt.u..].as_mut_ptr();
+            let f = if src_pixfmt == crate::Pixfmt::BGRA {
+                ARGBToNV12
+            } else {
+                ABGRToNV12
+            };
+            call_yuv!(f(
+                src.as_ptr(),
+                src_stride[0] as _,
+                dst_y,
+                dst_stride_y as _,
+                dst_uv,
+                dst_stride_uv as _,
+                src_width as _,
+                src_height as _,
+            ));
+        }
+        (crate::Pixfmt::BGRA, crate::Pixfmt::I444) | (crate::Pixfmt::RGBA, crate::Pixfmt::I444) => {
+            let dst_stride_y = dst_fmt.stride[0];
+            let dst_stride_u = dst_fmt.stride[1];
+            let dst_stride_v = dst_fmt.stride[2];
+            dst.resize(
+                align(dst_fmt.h) * (align(dst_stride_y) + align(dst_stride_u) + align(dst_stride_v)),
+                0,
+            );
+            let dst_y = dst.as_mut_ptr();
+            let dst_u = dst[dst_fmt.u..].as_mut_ptr();
+            let dst_v = dst[dst_fmt.v..].as_mut_ptr();
+            let src = if src_pixfmt == crate::Pixfmt::BGRA {
+                src
+            } else {
+                mid_data.resize(src.len(), 0);
+                call_yuv!(ABGRToARGB(
+                    src.as_ptr(),
+                    src_stride[0] as _,
+                    mid_data.as_mut_ptr(),
+                    src_stride[0] as _,
+                    src_width as _,
+                    src_height as _,
+                ));
+                mid_data
+            };
+            call_yuv!(ARGBToI444(
+                src.as_ptr(),
+                src_stride[0] as _,
+                dst_y,
+                dst_stride_y as _,
+                dst_u,
+                dst_stride_u as _,
+                dst_v,
+                dst_stride_v as _,
+                src_width as _,
+                src_height as _,
+            ));
+        }
+        _ => {
+            bail!(
+                "convert not support, {src_pixfmt:?} -> {:?}",
+                dst_fmt.pixfmt
+            );
+        }
+    }
+    Ok(())
+}

libs/scrap/src/common/dxgi.rs

@@ -1,10 +1,9 @@
-use crate::{common::TraitCapturer, dxgi};
+use crate::{common::TraitCapturer, dxgi, Pixfmt};
 use std::{
     io::{
         self,
         ErrorKind::{NotFound, TimedOut, WouldBlock},
     },
-    ops,
     time::Duration,
 };
 
@@ -15,10 +14,10 @@ pub struct Capturer {
 }
 
 impl Capturer {
-    pub fn new(display: Display, yuv: bool) -> io::Result<Capturer> {
+    pub fn new(display: Display) -> io::Result<Capturer> {
         let width = display.width();
         let height = display.height();
-        let inner = dxgi::Capturer::new(display.0, yuv)?;
+        let inner = dxgi::Capturer::new(display.0)?;
         Ok(Capturer {
             inner,
             width,
@@ -40,13 +39,9 @@ impl Capturer {
 }
 
 impl TraitCapturer for Capturer {
-    fn set_use_yuv(&mut self, use_yuv: bool) {
-        self.inner.set_use_yuv(use_yuv);
-    }
-
     fn frame<'a>(&'a mut self, timeout: Duration) -> io::Result<Frame<'a>> {
         match self.inner.frame(timeout.as_millis() as _) {
-            Ok(frame) => Ok(Frame(frame)),
+            Ok(frame) => Ok(Frame::new(frame, self.width, self.height)),
             Err(ref error) if error.kind() == TimedOut => Err(WouldBlock.into()),
             Err(error) => Err(error),
         }
@@ -61,12 +56,46 @@ impl TraitCapturer for Capturer {
     }
 }
 
-pub struct Frame<'a>(pub &'a [u8]);
+pub struct Frame<'a> {
+    data: &'a [u8],
+    width: usize,
+    height: usize,
+    stride: Vec<usize>,
+}
 
-impl<'a> ops::Deref for Frame<'a> {
-    type Target = [u8];
-    fn deref(&self) -> &[u8] {
-        self.0
+impl<'a> Frame<'a> {
+    pub fn new(data: &'a [u8], width: usize, height: usize) -> Self {
+        let stride0 = data.len() / height;
+        let mut stride = Vec::new();
+        stride.push(stride0);
+        Frame {
+            data,
+            width,
+            height,
+            stride,
+        }
+    }
+}
+
+impl<'a> crate::TraitFrame for Frame<'a> {
+    fn data(&self) -> &[u8] {
+        self.data
+    }
+
+    fn width(&self) -> usize {
+        self.width
+    }
+
+    fn height(&self) -> usize {
+        self.height
+    }
+
+    fn stride(&self) -> Vec<usize> {
+        self.stride.clone()
+    }
+
+    fn pixfmt(&self) -> Pixfmt {
+        Pixfmt::BGRA
     }
 }
 
@@ -134,9 +163,9 @@ impl CapturerMag {
         dxgi::mag::CapturerMag::is_supported()
     }
 
-    pub fn new(origin: (i32, i32), width: usize, height: usize, use_yuv: bool) -> io::Result<Self> {
+    pub fn new(origin: (i32, i32), width: usize, height: usize) -> io::Result<Self> {
         Ok(CapturerMag {
-            inner: dxgi::mag::CapturerMag::new(origin, width, height, use_yuv)?,
+            inner: dxgi::mag::CapturerMag::new(origin, width, height)?,
             data: Vec::new(),
         })
     }
@@ -151,13 +180,13 @@ impl CapturerMag {
 }
 
 impl TraitCapturer for CapturerMag {
-    fn set_use_yuv(&mut self, use_yuv: bool) {
-        self.inner.set_use_yuv(use_yuv)
-    }
-
     fn frame<'a>(&'a mut self, _timeout_ms: Duration) -> io::Result<Frame<'a>> {
         self.inner.frame(&mut self.data)?;
-        Ok(Frame(&self.data))
+        Ok(Frame::new(
+            &self.data,
+            self.inner.get_rect().1,
+            self.inner.get_rect().2,
+        ))
     }
 
     fn is_gdi(&self) -> bool {

libs/scrap/src/common/hwcodec.rs

@@ -1,6 +1,6 @@
 use crate::{
     codec::{base_bitrate, codec_thread_num, EncoderApi, EncoderCfg},
-    hw, ImageFormat, ImageRgb, HW_STRIDE_ALIGN,
+    hw, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN,
 };
 use hbb_common::{
     allow_err,
@@ -31,7 +31,6 @@ const DEFAULT_RC: RateControl = RC_DEFAULT;
 
 pub struct HwEncoder {
     encoder: Encoder,
-    yuv: Vec<u8>,
     pub format: DataFormat,
     pub pixfmt: AVPixelFormat,
     width: u32,
@@ -40,7 +39,7 @@ pub struct HwEncoder {
 }
 
 impl EncoderApi for HwEncoder {
-    fn new(cfg: EncoderCfg) -> ResultType<Self>
+    fn new(cfg: EncoderCfg, _i444: bool) -> ResultType<Self>
     where
         Self: Sized,
     {
@@ -78,7 +77,6 @@ impl EncoderApi for HwEncoder {
                 match Encoder::new(ctx.clone()) {
                     Ok(encoder) => Ok(HwEncoder {
                         encoder,
-                        yuv: vec![],
                         format,
                         pixfmt: ctx.pixfmt,
                         width: ctx.width as _,
@@ -118,8 +116,31 @@ impl EncoderApi for HwEncoder {
         }
     }
 
-    fn use_yuv(&self) -> bool {
-        false
+    fn yuvfmt(&self) -> crate::EncodeYuvFormat {
+        let pixfmt = if self.pixfmt == AVPixelFormat::AV_PIX_FMT_NV12 {
+            Pixfmt::NV12
+        } else {
+            Pixfmt::I420
+        };
+        let stride = self
+            .encoder
+            .linesize
+            .clone()
+            .drain(..)
+            .map(|i| i as usize)
+            .collect();
+        crate::EncodeYuvFormat {
+            pixfmt,
+            w: self.encoder.ctx.width as _,
+            h: self.encoder.ctx.height as _,
+            stride,
+            u: self.encoder.offset[0] as _,
+            v: if pixfmt == Pixfmt::NV12 {
+                0
+            } else {
+                self.encoder.offset[1] as _
+            },
+        }
     }
 
     fn set_quality(&mut self, quality: crate::codec::Quality) -> ResultType<()> {
@@ -145,29 +166,8 @@ impl HwEncoder {
         })
     }
 
-    pub fn encode(&mut self, bgra: &[u8]) -> ResultType<Vec<EncodeFrame>> {
-        match self.pixfmt {
-            AVPixelFormat::AV_PIX_FMT_YUV420P => hw::hw_bgra_to_i420(
-                self.encoder.ctx.width as _,
-                self.encoder.ctx.height as _,
-                &self.encoder.linesize,
-                &self.encoder.offset,
-                self.encoder.length,
-                bgra,
-                &mut self.yuv,
-            ),
-            AVPixelFormat::AV_PIX_FMT_NV12 => hw::hw_bgra_to_nv12(
-                self.encoder.ctx.width as _,
-                self.encoder.ctx.height as _,
-                &self.encoder.linesize,
-                &self.encoder.offset,
-                self.encoder.length,
-                bgra,
-                &mut self.yuv,
-            ),
-        }
-
-        match self.encoder.encode(&self.yuv) {
+    pub fn encode(&mut self, yuv: &[u8]) -> ResultType<Vec<EncodeFrame>> {
+        match self.encoder.encode(yuv) {
             Ok(v) => {
                 let mut data = Vec::<EncodeFrame>::new();
                 data.append(v);
@@ -245,7 +245,7 @@ impl HwDecoder {
     pub fn decode(&mut self, data: &[u8]) -> ResultType<Vec<HwDecoderImage>> {
         match self.decoder.decode(data) {
             Ok(v) => Ok(v.iter().map(|f| HwDecoderImage { frame: f }).collect()),
-            Err(_) => Ok(vec![]),
+            Err(e) => Err(anyhow!(e)),
         }
     }
 }
@@ -274,7 +274,7 @@ impl HwDecoderImage<'_> {
                 &mut rgb.raw as _,
                 i420,
                 HW_STRIDE_ALIGN,
-            ),
+            )?,
             AVPixelFormat::AV_PIX_FMT_YUV420P => {
                 hw::hw_i420_to(
                     rgb.fmt(),
@@ -287,10 +287,10 @@ impl HwDecoderImage<'_> {
                     frame.linesize[1] as _,
                     frame.linesize[2] as _,
                     &mut rgb.raw as _,
-                );
-                return Ok(());
+                )?;
             }
         }
+        Ok(())
     }
 
     pub fn bgra(&self, bgra: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> {

libs/scrap/src/common/linux.rs

@@ -11,10 +11,10 @@ pub enum Capturer {
 }
 
 impl Capturer {
-    pub fn new(display: Display, yuv: bool) -> io::Result<Capturer> {
+    pub fn new(display: Display) -> io::Result<Capturer> {
         Ok(match display {
-            Display::X11(d) => Capturer::X11(x11::Capturer::new(d, yuv)?),
-            Display::WAYLAND(d) => Capturer::WAYLAND(wayland::Capturer::new(d, yuv)?),
+            Display::X11(d) => Capturer::X11(x11::Capturer::new(d)?),
+            Display::WAYLAND(d) => Capturer::WAYLAND(wayland::Capturer::new(d)?),
         })
     }
 
@@ -34,13 +34,6 @@ impl Capturer {
 }
 
 impl TraitCapturer for Capturer {
-    fn set_use_yuv(&mut self, use_yuv: bool) {
-        match self {
-            Capturer::X11(d) => d.set_use_yuv(use_yuv),
-            Capturer::WAYLAND(d) => d.set_use_yuv(use_yuv),
-        }
-    }
-
     fn frame<'a>(&'a mut self, timeout: Duration) -> io::Result<Frame<'a>> {
         match self {
             Capturer::X11(d) => d.frame(timeout),

libs/scrap/src/common/mod.rs

@@ -1,5 +1,8 @@
 pub use self::vpxcodec::*;
-use hbb_common::message_proto::{video_frame, VideoFrame};
+use hbb_common::{
+    log,
+    message_proto::{video_frame, Chroma, VideoFrame},
+};
 use std::slice;
 
 cfg_if! {
@@ -96,8 +99,6 @@ pub fn would_block_if_equal(old: &mut Vec<u8>, b: &[u8]) -> std::io::Result<()>
 }
 
 pub trait TraitCapturer {
-    fn set_use_yuv(&mut self, use_yuv: bool);
-
     // We doesn't support
     #[cfg(not(any(target_os = "ios")))]
     fn frame<'a>(&'a mut self, timeout: std::time::Duration) -> std::io::Result<Frame<'a>>;
@@ -108,6 +109,36 @@ pub trait TraitCapturer {
     fn set_gdi(&mut self) -> bool;
 }
 
+pub trait TraitFrame {
+    fn data(&self) -> &[u8];
+
+    fn width(&self) -> usize;
+
+    fn height(&self) -> usize;
+
+    fn stride(&self) -> Vec<usize>;
+
+    fn pixfmt(&self) -> Pixfmt;
+}
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum Pixfmt {
+    BGRA,
+    RGBA,
+    I420,
+    NV12,
+    I444,
+}
+
+#[derive(Debug, Clone)]
+pub struct EncodeYuvFormat {
+    pub pixfmt: Pixfmt,
+    pub w: usize,
+    pub h: usize,
+    pub stride: Vec<usize>,
+    pub u: usize,
+    pub v: usize,
+}
+
 #[cfg(x11)]
 #[inline]
 pub fn is_x11() -> bool {
@@ -260,6 +291,7 @@ pub trait GoogleImage {
     fn height(&self) -> usize;
     fn stride(&self) -> Vec<i32>;
     fn planes(&self) -> Vec<*mut u8>;
+    fn chroma(&self) -> Chroma;
     fn get_bytes_per_row(w: usize, fmt: ImageFormat, stride: usize) -> usize {
         let bytes_per_pixel = match fmt {
             ImageFormat::Raw => 3,
@@ -278,8 +310,8 @@ pub trait GoogleImage {
         let stride = self.stride();
         let planes = self.planes();
         unsafe {
-            match rgb.fmt() {
-                ImageFormat::Raw => {
+            match (self.chroma(), rgb.fmt()) {
+                (Chroma::I420, ImageFormat::Raw) => {
                     super::I420ToRAW(
                         planes[0],
                         stride[0],
@@ -293,7 +325,7 @@ pub trait GoogleImage {
                         self.height() as _,
                     );
                 }
-                ImageFormat::ARGB => {
+                (Chroma::I420, ImageFormat::ARGB) => {
                     super::I420ToARGB(
                         planes[0],
                         stride[0],
@@ -307,7 +339,7 @@ pub trait GoogleImage {
                         self.height() as _,
                     );
                 }
-                ImageFormat::ABGR => {
+                (Chroma::I420, ImageFormat::ABGR) => {
                     super::I420ToABGR(
                         planes[0],
                         stride[0],
@@ -321,6 +353,36 @@ pub trait GoogleImage {
                         self.height() as _,
                     );
                 }
+                (Chroma::I444, ImageFormat::ARGB) => {
+                    super::I444ToARGB(
+                        planes[0],
+                        stride[0],
+                        planes[1],
+                        stride[1],
+                        planes[2],
+                        stride[2],
+                        rgb.raw.as_mut_ptr(),
+                        bytes_per_row as _,
+                        self.width() as _,
+                        self.height() as _,
+                    );
+                }
+                (Chroma::I444, ImageFormat::ABGR) => {
+                    super::I444ToABGR(
+                        planes[0],
+                        stride[0],
+                        planes[1],
+                        stride[1],
+                        planes[2],
+                        stride[2],
+                        rgb.raw.as_mut_ptr(),
+                        bytes_per_row as _,
+                        self.width() as _,
+                        self.height() as _,
+                    );
+                }
+                // (Chroma::I444, ImageFormat::Raw), new version libyuv have I444ToRAW
+                _ => log::error!("unsupported pixfmt:{:?}", self.chroma()),
             }
         }
     }

libs/scrap/src/common/quartz.rs

@@ -1,18 +1,16 @@
-use crate::quartz;
+use crate::{quartz, Pixfmt};
 use std::marker::PhantomData;
 use std::sync::{Arc, Mutex, TryLockError};
-use std::{io, mem, ops};
+use std::{io, mem};
 
 pub struct Capturer {
     inner: quartz::Capturer,
     frame: Arc<Mutex<Option<quartz::Frame>>>,
-    use_yuv: bool,
-    i420: Vec<u8>,
     saved_raw_data: Vec<u8>, // for faster compare and copy
 }
 
 impl Capturer {
-    pub fn new(display: Display, use_yuv: bool) -> io::Result<Capturer> {
+    pub fn new(display: Display) -> io::Result<Capturer> {
         let frame = Arc::new(Mutex::new(None));
 
         let f = frame.clone();
@@ -20,11 +18,7 @@ impl Capturer {
             display.0,
             display.width(),
             display.height(),
-            if use_yuv {
-                quartz::PixelFormat::YCbCr420Video
-            } else {
-                quartz::PixelFormat::Argb8888
-            },
+            quartz::PixelFormat::Argb8888,
             Default::default(),
             move |inner| {
                 if let Ok(mut f) = f.lock() {
@@ -37,8 +31,6 @@ impl Capturer {
         Ok(Capturer {
             inner,
             frame,
-            use_yuv,
-            i420: Vec::new(),
             saved_raw_data: Vec::new(),
         })
     }
@@ -53,10 +45,6 @@ impl Capturer {
 }
 
 impl crate::TraitCapturer for Capturer {
-    fn set_use_yuv(&mut self, use_yuv: bool) {
-        self.use_yuv = use_yuv;
-    }
-
     fn frame<'a>(&'a mut self, _timeout_ms: std::time::Duration) -> io::Result<Frame<'a>> {
         match self.frame.try_lock() {
             Ok(mut handle) => {
@@ -66,10 +54,13 @@ impl crate::TraitCapturer for Capturer {
                 match frame {
                     Some(mut frame) => {
                         crate::would_block_if_equal(&mut self.saved_raw_data, frame.inner())?;
-                        if self.use_yuv {
-                            frame.nv12_to_i420(self.width(), self.height(), &mut self.i420);
-                        }
-                        Ok(Frame(frame, PhantomData))
+                        frame.surface_to_bgra(self.height());
+                        Ok(Frame {
+                            frame,
+                            data: PhantomData,
+                            width: self.width(),
+                            height: self.height(),
+                        })
                     }
 
                     None => Err(io::ErrorKind::WouldBlock.into()),
@@ -83,12 +74,34 @@ impl crate::TraitCapturer for Capturer {
     }
 }
 
-pub struct Frame<'a>(pub quartz::Frame, PhantomData<&'a [u8]>);
+pub struct Frame<'a> {
+    frame: quartz::Frame,
+    data: PhantomData<&'a [u8]>,
+    width: usize,
+    height: usize,
+}
 
-impl<'a> ops::Deref for Frame<'a> {
-    type Target = [u8];
-    fn deref(&self) -> &[u8] {
-        &*self.0
+impl<'a> crate::TraitFrame for Frame<'a> {
+    fn data(&self) -> &[u8] {
+        &*self.frame
+    }
+
+    fn width(&self) -> usize {
+        self.width
+    }
+
+    fn height(&self) -> usize {
+        self.height
+    }
+
+    fn stride(&self) -> Vec<usize> {
+        let mut v = Vec::new();
+        v.push(self.frame.stride());
+        v
+    }
+
+    fn pixfmt(&self) -> Pixfmt {
+        Pixfmt::BGRA
     }
 }
 

libs/scrap/src/common/vpxcodec.rs

@@ -4,11 +4,11 @@
 
 use hbb_common::anyhow::{anyhow, Context};
 use hbb_common::log;
-use hbb_common::message_proto::{EncodedVideoFrame, EncodedVideoFrames, VideoFrame};
+use hbb_common::message_proto::{Chroma, EncodedVideoFrame, EncodedVideoFrames, VideoFrame};
 use hbb_common::ResultType;
 
 use crate::codec::{base_bitrate, codec_thread_num, EncoderApi, Quality};
-use crate::{GoogleImage, STRIDE_ALIGN};
+use crate::{EncodeYuvFormat, GoogleImage, Pixfmt, STRIDE_ALIGN};
 
 use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *};
 use crate::{generate_call_macro, generate_call_ptr_macro, Error, Result};
@@ -39,6 +39,8 @@ pub struct VpxEncoder {
     width: usize,
     height: usize,
     id: VpxVideoCodecId,
+    i444: bool,
+    yuvfmt: EncodeYuvFormat,
 }
 
 pub struct VpxDecoder {
@@ -46,7 +48,7 @@ pub struct VpxDecoder {
 }
 
 impl EncoderApi for VpxEncoder {
-    fn new(cfg: crate::codec::EncoderCfg) -> ResultType<Self>
+    fn new(cfg: crate::codec::EncoderCfg, i444: bool) -> ResultType<Self>
     where
         Self: Sized,
     {
@@ -98,6 +100,13 @@ impl EncoderApi for VpxEncoder {
                 } else {
                     c.rc_target_bitrate = base_bitrate;
                 }
+                // https://chromium.googlesource.com/webm/libvpx/+/refs/heads/main/vp9/common/vp9_enums.h#29
+                // https://chromium.googlesource.com/webm/libvpx/+/refs/heads/main/vp8/vp8_cx_iface.c#282
+                c.g_profile = if i444 && config.codec == VpxVideoCodecId::VP9 {
+                    1
+                } else {
+                    0
+                };
 
                 /*
                 The VPX encoder supports two-pass encoding for rate control purposes.
@@ -166,6 +175,8 @@ impl EncoderApi for VpxEncoder {
                     width: config.width as _,
                     height: config.height as _,
                     id: config.codec,
+                    i444,
+                    yuvfmt: Self::get_yuvfmt(config.width, config.height, i444),
                 })
             }
             _ => Err(anyhow!("encoder type mismatch")),
@@ -192,8 +203,8 @@ impl EncoderApi for VpxEncoder {
         }
     }
 
-    fn use_yuv(&self) -> bool {
-        true
+    fn yuvfmt(&self) -> crate::EncodeYuvFormat {
+        self.yuvfmt.clone()
     }
 
     fn set_quality(&mut self, quality: Quality) -> ResultType<()> {
@@ -219,14 +230,20 @@ impl EncoderApi for VpxEncoder {
 
 impl VpxEncoder {
     pub fn encode(&mut self, pts: i64, data: &[u8], stride_align: usize) -> Result<EncodeFrames> {
-        if 2 * data.len() < 3 * self.width * self.height {
+        let bpp = if self.i444 { 24 } else { 12 };
+        if data.len() < self.width * self.height * bpp / 8 {
             return Err(Error::FailedCall("len not enough".to_string()));
         }
+        let fmt = if self.i444 {
+            vpx_img_fmt::VPX_IMG_FMT_I444
+        } else {
+            vpx_img_fmt::VPX_IMG_FMT_I420
+        };
 
         let mut image = Default::default();
         call_vpx_ptr!(vpx_img_wrap(
             &mut image,
-            vpx_img_fmt::VPX_IMG_FMT_I420,
+            fmt,
             self.width as _,
             self.height as _,
             stride_align as _,
@@ -319,6 +336,34 @@ impl VpxEncoder {
 
         (q_min, q_max)
     }
+
+    fn get_yuvfmt(width: u32, height: u32, i444: bool) -> EncodeYuvFormat {
+        let mut img = Default::default();
+        let fmt = if i444 {
+            vpx_img_fmt::VPX_IMG_FMT_I444
+        } else {
+            vpx_img_fmt::VPX_IMG_FMT_I420
+        };
+        unsafe {
+            vpx_img_wrap(
+                &mut img,
+                fmt,
+                width as _,
+                height as _,
+                crate::STRIDE_ALIGN as _,
+                0x1 as _,
+            );
+        }
+        let pixfmt = if i444 { Pixfmt::I444 } else { Pixfmt::I420 };
+        EncodeYuvFormat {
+            pixfmt,
+            w: img.w as _,
+            h: img.h as _,
+            stride: img.stride.map(|s| s as usize).to_vec(),
+            u: img.planes[1] as usize - img.planes[0] as usize,
+            v: img.planes[2] as usize - img.planes[0] as usize,
+        }
+    }
 }
 
 impl Drop for VpxEncoder {
@@ -533,6 +578,13 @@ impl GoogleImage for Image {
     fn planes(&self) -> Vec<*mut u8> {
         self.inner().planes.iter().map(|p| *p as *mut u8).collect()
     }
+
+    fn chroma(&self) -> Chroma {
+        match self.inner().fmt {
+            vpx_img_fmt::VPX_IMG_FMT_I444 => Chroma::I444,
+            _ => Chroma::I420,
+        }
+    }
 }
 
 impl Drop for Image {

libs/scrap/src/common/wayland.rs

@@ -2,7 +2,7 @@ use crate::common::{x11::Frame, TraitCapturer};
 use crate::wayland::{capturable::*, *};
 use std::{io, sync::RwLock, time::Duration};
 
-pub struct Capturer(Display, Box<dyn Recorder>, bool, Vec<u8>);
+pub struct Capturer(Display, Box<dyn Recorder>, Vec<u8>);
 
 static mut IS_CURSOR_EMBEDDED: Option<bool> = None;
 
@@ -45,9 +45,9 @@ fn map_err<E: ToString>(err: E) -> io::Error {
 }
 
 impl Capturer {
-    pub fn new(display: Display, yuv: bool) -> io::Result<Capturer> {
+    pub fn new(display: Display) -> io::Result<Capturer> {
         let r = display.0.recorder(false).map_err(map_err)?;
-        Ok(Capturer(display, r, yuv, Default::default()))
+        Ok(Capturer(display, r, Default::default()))
     }
 
     pub fn width(&self) -> usize {
@@ -60,24 +60,10 @@ impl Capturer {
 }
 
 impl TraitCapturer for Capturer {
-    fn set_use_yuv(&mut self, use_yuv: bool) {
-        self.2 = use_yuv;
-    }
-
     fn frame<'a>(&'a mut self, timeout: Duration) -> io::Result<Frame<'a>> {
         match self.1.capture(timeout.as_millis() as _).map_err(map_err)? {
-            PixelProvider::BGR0(w, h, x) => Ok(Frame(if self.2 {
-                crate::common::bgra_to_i420(w as _, h as _, &x, &mut self.3);
-                &self.3[..]
-            } else {
-                x
-            })),
-            PixelProvider::RGB0(w, h, x) => Ok(Frame(if self.2 {
-                crate::common::rgba_to_i420(w as _, h as _, &x, &mut self.3);
-                &self.3[..]
-            } else {
-                x
-            })),
+            PixelProvider::BGR0(w, h, x) => Ok(Frame::new(x, crate::Pixfmt::BGRA, w, h)),
+            PixelProvider::RGB0(w, h, x) => Ok(Frame::new(x, crate::Pixfmt::RGBA, w,h)),
             PixelProvider::NONE => Err(std::io::ErrorKind::WouldBlock.into()),
             _ => Err(map_err("Invalid data")),
         }

libs/scrap/src/common/x11.rs

@@ -1,13 +1,13 @@
-use crate::{common::TraitCapturer, x11};
-use std::{io, ops, time::Duration};
+use crate::{common::TraitCapturer, x11, Pixfmt, TraitFrame};
+use std::{io, time::Duration};
 
 pub struct Capturer(x11::Capturer);
 
 pub const IS_CURSOR_EMBEDDED: bool = false;
 
 impl Capturer {
-    pub fn new(display: Display, yuv: bool) -> io::Result<Capturer> {
-        x11::Capturer::new(display.0, yuv).map(Capturer)
+    pub fn new(display: Display) -> io::Result<Capturer> {
+        x11::Capturer::new(display.0).map(Capturer)
     }
 
     pub fn width(&self) -> usize {
@@ -20,21 +20,53 @@ impl Capturer {
 }
 
 impl TraitCapturer for Capturer {
-    fn set_use_yuv(&mut self, use_yuv: bool) {
-        self.0.set_use_yuv(use_yuv);
-    }
-
     fn frame<'a>(&'a mut self, _timeout: Duration) -> io::Result<Frame<'a>> {
-        Ok(Frame(self.0.frame()?))
+        Ok(self.0.frame()?)
     }
 }
 
-pub struct Frame<'a>(pub &'a [u8]);
+pub struct Frame<'a> {
+    pub data: &'a [u8],
+    pub pixfmt: Pixfmt,
+    pub width: usize,
+    pub height: usize,
+    pub stride: Vec<usize>,
+}
 
-impl<'a> ops::Deref for Frame<'a> {
-    type Target = [u8];
-    fn deref(&self) -> &[u8] {
-        self.0
+impl<'a> Frame<'a> {
+    pub fn new(data: &'a [u8], pixfmt: Pixfmt, width: usize, height: usize) -> Self {
+        let stride0 = data.len() / height;
+        let mut stride = Vec::new();
+        stride.push(stride0);
+        Self {
+            data,
+            pixfmt,
+            width,
+            height,
+            stride,
+        }
+    }
+}
+
+impl<'a> TraitFrame for Frame<'a> {
+    fn data(&self) -> &[u8] {
+        self.data
+    }
+
+    fn width(&self) -> usize {
+        self.width
+    }
+
+    fn height(&self) -> usize {
+        self.height
+    }
+
+    fn stride(&self) -> Vec<usize> {
+        self.stride.clone()
+    }
+
+    fn pixfmt(&self) -> crate::Pixfmt {
+        self.pixfmt
     }
 }
 

libs/scrap/src/dxgi/gdi.rs

@@ -160,7 +160,7 @@ impl CapturerGDI {
                 stride,
                 self.width,
                 self.height,
-                180,
+                crate::RotationMode::kRotate180,
             );
             Ok(())
         }

libs/scrap/src/dxgi/mag.rs

@@ -245,9 +245,6 @@ pub struct CapturerMag {
     rect: RECT,
     width: usize,
     height: usize,
-
-    use_yuv: bool,
-    data: Vec<u8>,
 }
 
 impl Drop for CapturerMag {
@@ -262,12 +259,7 @@ impl CapturerMag {
         MagInterface::new().is_ok()
     }
 
-    pub(crate) fn new(
-        origin: (i32, i32),
-        width: usize,
-        height: usize,
-        use_yuv: bool,
-    ) -> Result<Self> {
+    pub(crate) fn new(origin: (i32, i32), width: usize, height: usize) -> Result<Self> {
         unsafe {
             let x = GetSystemMetrics(SM_XVIRTUALSCREEN);
             let y = GetSystemMetrics(SM_YVIRTUALSCREEN);
@@ -311,8 +303,6 @@ impl CapturerMag {
             },
             width,
             height,
-            use_yuv,
-            data: Vec::new(),
         };
 
         unsafe {
@@ -437,10 +427,6 @@ impl CapturerMag {
         Ok(s)
     }
 
-    pub(crate) fn set_use_yuv(&mut self, use_yuv: bool) {
-        self.use_yuv = use_yuv;
-    }
-
     pub(crate) fn exclude(&mut self, cls: &str, name: &str) -> Result<bool> {
         let name_c = CString::new(name)?;
         unsafe {
@@ -579,22 +565,9 @@ impl CapturerMag {
             ));
         }
 
-        if self.use_yuv {
-            self.data.resize(lock.1.len(), 0);
-            unsafe {
-                std::ptr::copy_nonoverlapping(&mut lock.1[0], &mut self.data[0], self.data.len());
-            }
-            crate::common::bgra_to_i420(
-                self.width as usize,
-                self.height as usize,
-                &self.data,
-                data,
-            );
-        } else {
-            data.resize(lock.1.len(), 0);
-            unsafe {
-                std::ptr::copy_nonoverlapping(&mut lock.1[0], &mut data[0], data.len());
-            }
+        data.resize(lock.1.len(), 0);
+        unsafe {
+            std::ptr::copy_nonoverlapping(&mut lock.1[0], &mut data[0], data.len());
         }
 
         Ok(())
@@ -651,7 +624,7 @@ mod tests {
     use super::*;
     #[test]
     fn test() {
-        let mut capture_mag = CapturerMag::new((0, 0), 1920, 1080, false).unwrap();
+        let mut capture_mag = CapturerMag::new((0, 0), 1920, 1080).unwrap();
         capture_mag.exclude("", "RustDeskPrivacyWindow").unwrap();
         std::thread::sleep(std::time::Duration::from_millis(1000 * 10));
         let mut data = Vec::new();

libs/scrap/src/dxgi/mod.rs

@@ -20,6 +20,8 @@ use winapi::{
     },
 };
 
+use crate::RotationMode::*;
+
 pub struct ComPtr<T>(*mut T);
 impl<T> ComPtr<T> {
     fn is_null(&self) -> bool {
@@ -45,8 +47,6 @@ pub struct Capturer {
     surface: ComPtr<IDXGISurface>,
     width: usize,
     height: usize,
-    use_yuv: bool,
-    yuv: Vec<u8>,
     rotated: Vec<u8>,
     gdi_capturer: Option<CapturerGDI>,
     gdi_buffer: Vec<u8>,
@@ -54,7 +54,7 @@ pub struct Capturer {
 }
 
 impl Capturer {
-    pub fn new(display: Display, use_yuv: bool) -> io::Result<Capturer> {
+    pub fn new(display: Display) -> io::Result<Capturer> {
         let mut device = ptr::null_mut();
         let mut context = ptr::null_mut();
         let mut duplication = ptr::null_mut();
@@ -148,8 +148,6 @@ impl Capturer {
             width: display.width() as usize,
             height: display.height() as usize,
             display,
-            use_yuv,
-            yuv: Vec::new(),
             rotated: Vec::new(),
             gdi_capturer,
             gdi_buffer: Vec::new(),
@@ -157,10 +155,6 @@ impl Capturer {
         })
     }
 
-    pub fn set_use_yuv(&mut self, use_yuv: bool) {
-        self.use_yuv = use_yuv;
-    }
-
     pub fn is_gdi(&self) -> bool {
         self.gdi_capturer.is_some()
     }
@@ -259,10 +253,10 @@ impl Capturer {
                     self.unmap();
                     let r = self.load_frame(timeout)?;
                     let rotate = match self.display.rotation() {
-                        DXGI_MODE_ROTATION_IDENTITY | DXGI_MODE_ROTATION_UNSPECIFIED => 0,
-                        DXGI_MODE_ROTATION_ROTATE90 => 90,
-                        DXGI_MODE_ROTATION_ROTATE180 => 180,
-                        DXGI_MODE_ROTATION_ROTATE270 => 270,
+                        DXGI_MODE_ROTATION_IDENTITY | DXGI_MODE_ROTATION_UNSPECIFIED => kRotate0,
+                        DXGI_MODE_ROTATION_ROTATE90 => kRotate90,
+                        DXGI_MODE_ROTATION_ROTATE180 => kRotate180,
+                        DXGI_MODE_ROTATION_ROTATE270 => kRotate270,
                         _ => {
                             return Err(io::Error::new(
                                 io::ErrorKind::Other,
@@ -270,7 +264,7 @@ impl Capturer {
                             ));
                         }
                     };
-                    if rotate == 0 {
+                    if rotate == kRotate0 {
                         slice::from_raw_parts(r.0, r.1 as usize * self.height)
                     } else {
                         self.rotated.resize(self.width * self.height * 4, 0);
@@ -279,12 +273,12 @@ impl Capturer {
                             r.1,
                             self.rotated.as_mut_ptr(),
                             4 * self.width as i32,
-                            if rotate == 180 {
+                            if rotate == kRotate180 {
                                 self.width
                             } else {
                                 self.height
                             } as _,
-                            if rotate != 180 {
+                            if rotate != kRotate180 {
                                 self.width
                             } else {
                                 self.height
@@ -295,19 +289,7 @@ impl Capturer {
                     }
                 }
             };
-            Ok({
-                if self.use_yuv {
-                    crate::common::bgra_to_i420(
-                        self.width as usize,
-                        self.height as usize,
-                        &result,
-                        &mut self.yuv,
-                    );
-                    &self.yuv[..]
-                } else {
-                    result
-                }
-            })
+            Ok(result)
         }
     }
 

libs/scrap/src/quartz/frame.rs

@@ -5,8 +5,8 @@ use super::ffi::*;
 pub struct Frame {
     surface: IOSurfaceRef,
     inner: &'static [u8],
-    i420: *mut u8,
-    i420_len: usize,
+    bgra: Vec<u8>,
+    bgra_stride: usize,
 }
 
 impl Frame {
@@ -24,8 +24,8 @@ impl Frame {
         Frame {
             surface,
             inner,
-            i420: ptr::null_mut(),
-            i420_len: 0,
+            bgra: Vec::new(),
+            bgra_stride: 0,
         }
     }
 
@@ -34,23 +34,20 @@ impl Frame {
         self.inner
     }
 
-    pub fn nv12_to_i420<'a>(&'a mut self, w: usize, h: usize, i420: &'a mut Vec<u8>) {
+    pub fn stride(&self) -> usize {
+        self.bgra_stride
+    }
+
+    pub fn surface_to_bgra<'a>(&'a mut self, h: usize) {
         unsafe {
             let plane0 = IOSurfaceGetBaseAddressOfPlane(self.surface, 0);
-            let stride0 = IOSurfaceGetBytesPerRowOfPlane(self.surface, 0);
-            let plane1 = IOSurfaceGetBaseAddressOfPlane(self.surface, 1);
-            let stride1 = IOSurfaceGetBytesPerRowOfPlane(self.surface, 1);
-            crate::common::nv12_to_i420(
+            self.bgra_stride = IOSurfaceGetBytesPerRowOfPlane(self.surface, 0);
+            self.bgra.resize(self.bgra_stride * h, 0);
+            std::ptr::copy_nonoverlapping(
                 plane0 as _,
-                stride0 as _,
-                plane1 as _,
-                stride1 as _,
-                w,
-                h,
-                i420,
+                self.bgra.as_mut_ptr(),
+                self.bgra_stride * h,
             );
-            self.i420 = i420.as_mut_ptr() as _;
-            self.i420_len = i420.len();
         }
     }
 }
@@ -58,14 +55,7 @@ impl Frame {
 impl ops::Deref for Frame {
     type Target = [u8];
     fn deref<'a>(&'a self) -> &'a [u8] {
-        if self.i420.is_null() {
-            self.inner
-        } else {
-            unsafe {
-                let inner = slice::from_raw_parts(self.i420 as *const u8, self.i420_len);
-                inner
-            }
-        }
+        &self.bgra
     }
 }
 

libs/scrap/src/x11/capturer.rs

@@ -2,6 +2,8 @@ use std::{io, ptr, slice};
 
 use hbb_common::libc;
 
+use crate::Frame;
+
 use super::ffi::*;
 use super::Display;
 
@@ -12,13 +14,11 @@ pub struct Capturer {
     buffer: *const u8,
 
     size: usize,
-    use_yuv: bool,
-    yuv: Vec<u8>,
     saved_raw_data: Vec<u8>, // for faster compare and copy
 }
 
 impl Capturer {
-    pub fn new(display: Display, use_yuv: bool) -> io::Result<Capturer> {
+    pub fn new(display: Display) -> io::Result<Capturer> {
         // Calculate dimensions.
 
         let pixel_width = 4;
@@ -67,17 +67,11 @@ impl Capturer {
             xcbid,
             buffer,
             size,
-            use_yuv,
-            yuv: Vec::new(),
             saved_raw_data: Vec::new(),
         };
         Ok(c)
     }
 
-    pub fn set_use_yuv(&mut self, use_yuv: bool) {
-        self.use_yuv = use_yuv;
-    }
-
     pub fn display(&self) -> &Display {
         &self.display
     }
@@ -103,16 +97,13 @@ impl Capturer {
         }
     }
 
-    pub fn frame<'b>(&'b mut self) -> std::io::Result<&'b [u8]> {
+    pub fn frame<'b>(&'b mut self) -> std::io::Result<Frame> {
         self.get_image();
         let result = unsafe { slice::from_raw_parts(self.buffer, self.size) };
         crate::would_block_if_equal(&mut self.saved_raw_data, result)?;
-        Ok(if self.use_yuv {
-            crate::common::bgra_to_i420(self.display.w(), self.display.h(), &result, &mut self.yuv);
-            &self.yuv[..]
-        } else {
-            result
-        })
+        Ok(
+            Frame::new(result, crate::Pixfmt::BGRA, self.display.w(), self.display.h())
+       )
     }
 }