add zero copy mode hareware codec for windows (#6778)

Signed-off-by: 21pages <pages21@163.com>

src/server/video_service.rs

@@ -42,15 +42,18 @@ use hbb_common::{
         Mutex as TokioMutex,
     },
 };
+#[cfg(feature = "gpucodec")]
+use scrap::gpucodec::{GpuEncoder, GpuEncoderConfig};
+#[cfg(feature = "hwcodec")]
+use scrap::hwcodec::{HwEncoder, HwEncoderConfig};
 #[cfg(not(windows))]
 use scrap::Capturer;
 use scrap::{
     aom::AomEncoderConfig,
-    codec::{Encoder, EncoderCfg, HwEncoderConfig, Quality},
-    convert_to_yuv,
+    codec::{Encoder, EncoderCfg, EncodingUpdate, Quality},
     record::{Recorder, RecorderContext},
     vpxcodec::{VpxEncoderConfig, VpxVideoCodecId},
-    CodecName, Display, Frame, TraitCapturer, TraitFrame,
+    CodecName, Display, Frame, TraitCapturer,
 };
 #[cfg(windows)]
 use std::sync::Once;
@@ -363,6 +366,7 @@ fn get_capturer(current: usize, portable_service_running: bool) -> ResultType<Ca
 }
 
 fn run(vs: VideoService) -> ResultType<()> {
+    let _raii = Raii::new(vs.idx);
     // Wayland only support one video capturer for now. It is ok to call ensure_inited() here.
     //
     // ensure_inited() is needed because clear() may be called.
@@ -391,21 +395,33 @@ fn run(vs: VideoService) -> ResultType<()> {
     video_qos.refresh(None);
     let mut spf;
     let mut quality = video_qos.quality();
-    let abr = VideoQoS::abr_enabled();
-    log::info!("initial quality: {quality:?}, abr enabled: {abr}");
-    let codec_name = Encoder::negotiated_codec();
-    let recorder = get_recorder(c.width, c.height, &codec_name);
-    let last_recording = recorder.lock().unwrap().is_some() || video_qos.record();
+    let record_incoming = !Config::get_option("allow-auto-record-incoming").is_empty();
+    let client_record = video_qos.record();
     drop(video_qos);
-    let encoder_cfg = get_encoder_config(&c, quality, last_recording);
-
+    let encoder_cfg = get_encoder_config(
+        &c,
+        display_idx,
+        quality,
+        client_record || record_incoming,
+        last_portable_service_running,
+    );
+    Encoder::set_fallback(&encoder_cfg);
+    let codec_name = Encoder::negotiated_codec();
+    let recorder = get_recorder(c.width, c.height, &codec_name, record_incoming);
     let mut encoder;
     let use_i444 = Encoder::use_i444(&encoder_cfg);
     match Encoder::new(encoder_cfg.clone(), use_i444) {
         Ok(x) => encoder = x,
         Err(err) => bail!("Failed to create encoder: {}", err),
     }
+    #[cfg(feature = "gpucodec")]
+    c.set_output_texture(encoder.input_texture());
     VIDEO_QOS.lock().unwrap().store_bitrate(encoder.bitrate());
+    VIDEO_QOS
+        .lock()
+        .unwrap()
+        .set_support_abr(display_idx, encoder.support_abr());
+    log::info!("initial quality: {quality:?}");
 
     if sp.is_option_true(OPTION_REFRESH) {
         sp.set_option_bool(OPTION_REFRESH, false);
@@ -439,8 +455,7 @@ fn run(vs: VideoService) -> ResultType<()> {
             allow_err!(encoder.set_quality(quality));
             video_qos.store_bitrate(encoder.bitrate());
         }
-        let recording = recorder.lock().unwrap().is_some() || video_qos.record();
-        if recording != last_recording {
+        if client_record != video_qos.record() {
             bail!("SWITCH");
         }
         drop(video_qos);
@@ -459,6 +474,11 @@ fn run(vs: VideoService) -> ResultType<()> {
         if Encoder::use_i444(&encoder_cfg) != use_i444 {
             bail!("SWITCH");
         }
+        #[cfg(all(windows, feature = "gpucodec"))]
+        if c.is_gdi() && (codec_name == CodecName::H264GPU || codec_name == CodecName::H265GPU) {
+            log::info!("changed to gdi when using gpucodec");
+            bail!("SWITCH");
+        }
         check_privacy_mode_changed(&sp, c.privacy_mode_id)?;
         #[cfg(windows)]
         {
@@ -482,7 +502,7 @@ fn run(vs: VideoService) -> ResultType<()> {
             Ok(frame) => {
                 let time = now - start;
                 let ms = (time.as_secs() * 1000 + time.subsec_millis() as u64) as i64;
-                if frame.data().len() != 0 {
+                if frame.valid() {
                     let send_conn_ids = handle_one_frame(
                         display_idx,
                         &sp,
@@ -533,6 +553,8 @@ fn run(vs: VideoService) -> ResultType<()> {
                 }
             }
             Err(err) => {
+                // Get display information again immediately after error.
+                crate::display_service::check_displays_changed().ok();
                 // This check may be redundant, but it is better to be safe.
                 // The previous check in `sp.is_option_true(OPTION_REFRESH)` block may be enough.
                 try_broadcast_display_changed(&sp, display_idx, &c)?;
@@ -576,36 +598,159 @@ fn run(vs: VideoService) -> ResultType<()> {
     Ok(())
 }
 
-fn get_encoder_config(c: &CapturerInfo, quality: Quality, recording: bool) -> EncoderCfg {
+struct Raii(usize);
+
+impl Raii {
+    fn new(display_idx: usize) -> Self {
+        Raii(display_idx)
+    }
+}
+
+impl Drop for Raii {
+    fn drop(&mut self) {
+        #[cfg(feature = "gpucodec")]
+        GpuEncoder::set_not_use(self.0, false);
+        VIDEO_QOS.lock().unwrap().set_support_abr(self.0, true);
+    }
+}
+
+fn get_encoder_config(
+    c: &CapturerInfo,
+    _display_idx: usize,
+    quality: Quality,
+    record: bool,
+    _portable_service: bool,
+) -> EncoderCfg {
+    #[cfg(all(windows, feature = "gpucodec"))]
+    if _portable_service || c.is_gdi() {
+        log::info!("gdi:{}, portable:{}", c.is_gdi(), _portable_service);
+        GpuEncoder::set_not_use(_display_idx, true);
+    }
+    #[cfg(feature = "gpucodec")]
+    Encoder::update(EncodingUpdate::Check);
     // https://www.wowza.com/community/t/the-correct-keyframe-interval-in-obs-studio/95162
-    let keyframe_interval = if recording { Some(240) } else { None };
-    match Encoder::negotiated_codec() {
-        scrap::CodecName::H264(name) | scrap::CodecName::H265(name) => {
-            EncoderCfg::HW(HwEncoderConfig {
-                name,
-                width: c.width,
-                height: c.height,
-                quality,
+    let keyframe_interval = if record { Some(240) } else { None };
+    let negotiated_codec = Encoder::negotiated_codec();
+    match negotiated_codec.clone() {
+        CodecName::H264GPU | CodecName::H265GPU => {
+            #[cfg(feature = "gpucodec")]
+            if let Some(feature) = GpuEncoder::try_get(&c.device(), negotiated_codec.clone()) {
+                EncoderCfg::GPU(GpuEncoderConfig {
+                    device: c.device(),
+                    width: c.width,
+                    height: c.height,
+                    quality,
+                    feature,
+                    keyframe_interval,
+                })
+            } else {
+                handle_hw_encoder(
+                    negotiated_codec.clone(),
+                    c.width,
+                    c.height,
+                    quality as _,
+                    keyframe_interval,
+                )
+            }
+            #[cfg(not(feature = "gpucodec"))]
+            handle_hw_encoder(
+                negotiated_codec.clone(),
+                c.width,
+                c.height,
+                quality as _,
                 keyframe_interval,
-            })
+            )
         }
-        name @ (scrap::CodecName::VP8 | scrap::CodecName::VP9) => {
-            EncoderCfg::VPX(VpxEncoderConfig {
-                width: c.width as _,
-                height: c.height as _,
-                quality,
-                codec: if name == scrap::CodecName::VP8 {
-                    VpxVideoCodecId::VP8
-                } else {
-                    VpxVideoCodecId::VP9
-                },
-                keyframe_interval,
-            })
-        }
-        scrap::CodecName::AV1 => EncoderCfg::AOM(AomEncoderConfig {
+        CodecName::H264HW(_name) | CodecName::H265HW(_name) => handle_hw_encoder(
+            negotiated_codec.clone(),
+            c.width,
+            c.height,
+            quality as _,
+            keyframe_interval,
+        ),
+        name @ (CodecName::VP8 | CodecName::VP9) => EncoderCfg::VPX(VpxEncoderConfig {
             width: c.width as _,
             height: c.height as _,
             quality,
+            codec: if name == CodecName::VP8 {
+                VpxVideoCodecId::VP8
+            } else {
+                VpxVideoCodecId::VP9
+            },
+            keyframe_interval,
+        }),
+        CodecName::AV1 => EncoderCfg::AOM(AomEncoderConfig {
+            width: c.width as _,
+            height: c.height as _,
+            quality,
+            keyframe_interval,
+        }),
+    }
+}
+
+fn handle_hw_encoder(
+    _name: CodecName,
+    width: usize,
+    height: usize,
+    quality: Quality,
+    keyframe_interval: Option<usize>,
+) -> EncoderCfg {
+    let f = || {
+        #[cfg(feature = "hwcodec")]
+        match _name {
+            CodecName::H264GPU | CodecName::H265GPU => {
+                if !scrap::codec::enable_hwcodec_option() {
+                    return Err(());
+                }
+                let is_h265 = _name == CodecName::H265GPU;
+                let best = HwEncoder::best();
+                if let Some(h264) = best.h264 {
+                    if !is_h265 {
+                        return Ok(EncoderCfg::HW(HwEncoderConfig {
+                            name: h264.name,
+                            width,
+                            height,
+                            quality,
+                            keyframe_interval,
+                        }));
+                    }
+                }
+                if let Some(h265) = best.h265 {
+                    if is_h265 {
+                        return Ok(EncoderCfg::HW(HwEncoderConfig {
+                            name: h265.name,
+                            width,
+                            height,
+                            quality,
+                            keyframe_interval,
+                        }));
+                    }
+                }
+            }
+            CodecName::H264HW(name) | CodecName::H265HW(name) => {
+                return Ok(EncoderCfg::HW(HwEncoderConfig {
+                    name,
+                    width,
+                    height,
+                    quality,
+                    keyframe_interval,
+                }));
+            }
+            _ => {
+                return Err(());
+            }
+        };
+
+        Err(())
+    };
+
+    match f() {
+        Ok(cfg) => cfg,
+        _ => EncoderCfg::VPX(VpxEncoderConfig {
+            width: width as _,
+            height: height as _,
+            quality,
+            codec: VpxVideoCodecId::VP9,
             keyframe_interval,
         }),
     }
@@ -615,8 +760,9 @@ fn get_recorder(
     width: usize,
     height: usize,
     codec_name: &CodecName,
+    record_incoming: bool,
 ) -> Arc<Mutex<Option<Recorder>>> {
-    let recorder = if !Config::get_option("allow-auto-record-incoming").is_empty() {
+    let recorder = if record_incoming {
         use crate::hbbs_http::record_upload;
 
         let tx = if record_upload::is_enable() {
@@ -678,18 +824,26 @@ fn handle_one_frame(
         Ok(())
     })?;
 
+    let frame = frame.to(encoder.yuvfmt(), yuv, mid_data)?;
     let mut send_conn_ids: HashSet<i32> = Default::default();
-    convert_to_yuv(&frame, encoder.yuvfmt(), yuv, mid_data)?;
-    if let Ok(mut vf) = encoder.encode_to_message(yuv, ms) {
-        vf.display = display as _;
-        let mut msg = Message::new();
-        msg.set_video_frame(vf);
-        recorder
-            .lock()
-            .unwrap()
-            .as_mut()
-            .map(|r| r.write_message(&msg));
-        send_conn_ids = sp.send_video_frame(msg);
+    match encoder.encode_to_message(frame, ms) {
+        Ok(mut vf) => {
+            vf.display = display as _;
+            let mut msg = Message::new();
+            msg.set_video_frame(vf);
+            recorder
+                .lock()
+                .unwrap()
+                .as_mut()
+                .map(|r| r.write_message(&msg));
+            send_conn_ids = sp.send_video_frame(msg);
+        }
+        Err(e) => match e.to_string().as_str() {
+            scrap::codec::ENCODE_NEED_SWITCH => {
+                bail!("SWITCH");
+            }
+            _ => {}
+        },
     }
     Ok(send_conn_ids)
 }