scrap: load the D3D compiler securely from System32

Signed-off-by: 21pages <sunboeasy@gmail.com>

libs/scrap/src/dxgi/hdr.rs

@@ -49,7 +49,7 @@ use winapi::{
     um::{
         d3d11::*,
         d3dcommon::{ID3DBlob, ID3DInclude, D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST, D3D_SHADER_MACRO},
-        libloaderapi::{GetProcAddress, LoadLibraryW},
+        libloaderapi::{GetProcAddress, LoadLibraryExW, LOAD_LIBRARY_SEARCH_SYSTEM32},
         unknwnbase::IUnknown,
         wingdi::{
             DISPLAYCONFIG_DEVICE_INFO_GET_SOURCE_NAME, DISPLAYCONFIG_DEVICE_INFO_HEADER,
@@ -495,7 +495,7 @@ fn load_d3d_compile() -> io::Result<D3DCompileFn> {
 
 unsafe fn find_d3d_compile() -> Result<D3DCompileFn, String> {
     let name: Vec<u16> = "d3dcompiler_47.dll\0".encode_utf16().collect();
-    let module = LoadLibraryW(name.as_ptr());
+    let module = LoadLibraryExW(name.as_ptr(), ptr::null_mut(), LOAD_LIBRARY_SEARCH_SYSTEM32);
     if module.is_null() {
         return Err("d3dcompiler_47.dll not available".into());
     }

libs/scrap/src/dxgi/mod.rs

@@ -211,6 +211,10 @@ impl Capturer {
                 if hres == S_OK {
                     return hres;
                 }
+                hbb_common::log::warn!(
+                    "HDR DuplicateOutput1 failed: hr={:#x}, fallback=DuplicateOutput",
+                    hres as u32
+                );
             }
         }
         (*display.inner.0).DuplicateOutput(device as *mut _, duplication)
@@ -243,9 +247,8 @@ impl Capturer {
     }
 
     // Drops the tone-map and re-duplicates the output the legacy way, so DXGI
-    // hands over clipped BGRA8 (the pre-HDR behaviour) and the session stays on
-    // DXGI instead of being switched to GDI by the capture loop, which treats
-    // any other error that way. The caller sees WouldBlock and asks again.
+    // hands over clipped BGRA8 (the pre-HDR behaviour). If re-duplication fails,
+    // switch to GDI before returning. The caller sees WouldBlock and asks again.
     unsafe fn abandon_tonemap<T>(&mut self, err: io::Error) -> io::Result<T> {
         if hdr::is_permanent(&err) {
             hdr::UNAVAILABLE.store(true, std::sync::atomic::Ordering::Relaxed);
@@ -255,9 +258,15 @@ impl Capturer {
         (*self.duplication.0).ReleaseFrame();
         self.duplication = ComPtr(ptr::null_mut());
         let mut duplication = ptr::null_mut();
-        wrap_hresult(
+        let result = wrap_hresult(
             (*self.display.inner.0).DuplicateOutput(self.device.0 as *mut _, &mut duplication),
-        )?;
+        );
+        if let Err(err) = result {
+            if self.set_gdi() {
+                return Err(io::ErrorKind::WouldBlock.into());
+            }
+            return Err(err);
+        }
         self.duplication = ComPtr(duplication);
         let mut desc: DXGI_OUTDUPL_DESC = mem::zeroed();
         (*duplication).GetDesc(&mut desc);
@@ -418,6 +427,9 @@ impl Capturer {
     }
 
     unsafe fn load_frame(&mut self, timeout: UINT) -> io::Result<(*const u8, i32)> {
+        if self.duplication.0.is_null() {
+            return Err(io::ErrorKind::AddrNotAvailable.into());
+        }
         let mut frame = ptr::null_mut();
         #[allow(invalid_value)]
         let mut info = mem::MaybeUninit::uninit().assume_init();
@@ -677,6 +689,9 @@ impl Capturer {
     }
 
     fn unmap(&self) {
+        if self.duplication.0.is_null() {
+            return;
+        }
         unsafe {
             (*self.duplication.0).ReleaseFrame();
             if self.fastlane {