WebClient: 3.44 webcodecs offline (#15722)

* feat(web): zero-readback WebCodecs video path

Decoded VideoFrames from js/src/webcodecs.js are handed to Flutter via
window.onVideoFrame and imported GPU-side with createImageFromTextureSource;
any failure unregisters the hook so the JS side falls back to RGBA readback.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(web): load bundled terminal font when Google CDNs are unreachable

In air-gapped deployments GoogleFonts.robotoMono() cannot download the
terminal font; when index.html signals offline mode, load the copy bundled
with the web app under the family name google_fonts registers.

Part of the fix for rustdesk/rustdesk-server-pro#996.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* ci: bump windows arm64 to Flutter 3.44.8, add web build patch script

apply_flutter_3.44_web_patches.sh prepares a 3.44.x web build on top of the
shared source patches: qr_code_scanner's web impl needs dart:ui_web for the
removed platformViewRegistry, and flutter/web/fonts is refreshed to the font
paths the 3.44 engine requests. The disabled build-rustdesk-web job runs it
automatically once FLUTTER_VERSION moves to 3.44.x, and version-guarded
'Patch flutter' steps no longer fail when the guard does not match.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(web): prevent stale WebCodecs frames across sessions

Signed-off-by: fufesou <linlong1266@gmail.com>

* fix(web): harden WebCodecs reconnect and Flutter 3.44 patches

Signed-off-by: fufesou <linlong1266@gmail.com>

* fix(ci): harden Flutter 3.44 patch input validation

Validate required files before checking patch state,
parameterize the theme-range validator, and prevent
missing inputs from satisfying NO_MATCHES checks.

Signed-off-by: fufesou <linlong1266@gmail.com>

* Remove unused code

Signed-off-by: fufesou <linlong1266@gmail.com>

* fix(web): retry font loading and dispose stale decoded images

Signed-off-by: fufesou <linlong1266@gmail.com>

* remove unused code

Signed-off-by: fufesou <linlong1266@gmail.com>

* fix(web): Bad state: RenderBox was not laid out

Signed-off-by: fufesou <linlong1266@gmail.com>

---------

Signed-off-by: fufesou <linlong1266@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: fufesou <linlong1266@gmail.com>

flutter/lib/main.dart

@@ -588,7 +588,8 @@ _registerEventHandler() {
 
 Widget keyListenerBuilder(BuildContext context, Widget? child) {
   return RawKeyboardListener(
-    focusNode: FocusNode(),
+    // `skipTraversal: isWeb` is to fix "Bad state: RenderBox was not laid out: minified:aeL#c19e4"
+    focusNode: FocusNode(skipTraversal: isWeb),
     child: child ?? Container(),
     onKey: (RawKeyEvent event) {
       if (event.logicalKey == LogicalKeyboardKey.shiftLeft) {

flutter/lib/mobile/pages/terminal_page.dart

@@ -10,6 +10,8 @@ import 'package:flutter_hbb/models/model.dart';
 import 'package:flutter_hbb/models/platform_model.dart';
 import 'package:flutter_hbb/models/terminal_model.dart';
 import 'package:flutter_hbb/mobile/terminal_keyboard_utils.dart';
+import 'package:flutter_hbb/web/dummy.dart'
+    if (dart.library.html) 'package:flutter_hbb/web/terminal_font.dart';
 import 'package:google_fonts/google_fonts.dart';
 import 'package:xterm/xterm.dart';
 import '../../desktop/pages/terminal_connection_manager.dart';
@@ -67,6 +69,10 @@ class _TerminalPageState extends State<TerminalPage>
     super.initState();
     WidgetsBinding.instance.addObserver(this);
 
+    if (isWeb) {
+      loadLocalTerminalFontIfNeeded();
+    }
+
     debugPrint(
         '[TerminalPage] Initializing terminal ${widget.terminalId} for peer ${widget.id}');
 

flutter/lib/models/model.dart

@@ -1952,6 +1952,12 @@ class ImageModel with ChangeNotifier {
     platformFFI.nextRgba(sessionId, display);
   }
 
+  // web only: image already created from a decoded WebCodecs frame
+  Future<void> onImage(
+      int display, ui.Image image, bool Function() isCurrentSession) async {
+    await update(image, isCurrentSession: isCurrentSession);
+  }
+
   decodeAndUpdate(int display, Uint8List rgba) async {
     final pid = parent.target?.id;
     final rect = parent.target?.ffiModel.pi.getDisplayRect(display);
@@ -1963,11 +1969,16 @@ class ImageModel with ChangeNotifier {
           ? ui.PixelFormat.rgba8888
           : ui.PixelFormat.bgra8888,
     );
-    if (parent.target?.id != pid) return;
+    if (parent.target?.id != pid) {
+      image?.dispose();
+      return;
+    }
     await update(image);
   }
 
-  update(ui.Image? image) async {
+  Future<void> update(ui.Image? image,
+      {bool Function()? isCurrentSession}) async {
+    if (_disposeIfStale(image, isCurrentSession)) return;
     if (_image == null && image != null) {
       if (isDesktop || isWebDesktop) {
         await parent.target?.canvasModel.updateViewStyle();
@@ -1978,11 +1989,19 @@ class ImageModel with ChangeNotifier {
         await initializeCursorAndCanvas(parent.target!);
       }
     }
+    if (_disposeIfStale(image, isCurrentSession)) return;
     _image?.dispose();
     _image = image;
     if (image != null) notifyListeners();
   }
 
+  bool _disposeIfStale(ui.Image? image, bool Function()? isCurrentSession) {
+    if (image == null || isCurrentSession == null) return false;
+    if (isCurrentSession()) return false;
+    image.dispose();
+    return true;
+  }
+
   // mobile only
   double get maxScale {
     if (_image == null) return 1.5;
@@ -3853,6 +3872,15 @@ class FFI {
         onEvent2UIRgba();
         imageModel.onRgba(display, data);
       });
+      platformFFI.setVideoFrameCallback((int display, ui.Image image,
+          bool Function() isCurrentSession) async {
+        if (!isCurrentSession()) {
+          image.dispose();
+          return;
+        }
+        await onEvent2UIRgba();
+        await imageModel.onImage(display, image, isCurrentSession);
+      });
       this.id = id;
       return;
     }
@@ -3940,7 +3968,7 @@ class FFI {
     this.id = id;
   }
 
-  void onEvent2UIRgba() async {
+  Future<void> onEvent2UIRgba() async {
     if (ffiModel.waitForImageDialogShow.isTrue) {
       ffiModel.waitForImageDialogShow.value = false;
       ffiModel.waitForImageTimer?.cancel();
@@ -3996,6 +4024,9 @@ class FFI {
   /// Close the remote session.
   Future<void> close({bool closeSession = true}) async {
     closed = true;
+    if (isWeb) {
+      platformFFI.clearVideoFrameCallback();
+    }
     chatModel.close();
     // Close all terminal models
     for (final model in _terminalModels.values) {

flutter/lib/models/native_model.dart

@@ -1,6 +1,7 @@
 import 'dart:convert';
 import 'dart:ffi';
 import 'dart:io';
+import 'dart:ui' as ui;
 
 import 'package:device_info_plus/device_info_plus.dart';
 import 'package:external_path/external_path.dart';
@@ -283,6 +284,12 @@ class PlatformFFI {
 
   void setRgbaCallback(void Function(int, Uint8List) fun) async {}
 
+  // web only, decoded WebCodecs frames arriving as ready-made images
+  void setVideoFrameCallback(
+      Future<void> Function(int, ui.Image, bool Function()) fun) {}
+
+  void clearVideoFrameCallback() {}
+
   void startDesktopWebListener() {}
 
   void stopDesktopWebListener() {}

flutter/lib/models/web_model.dart

@@ -2,14 +2,18 @@
 
 import 'dart:convert';
 import 'dart:js_interop';
+import 'dart:js_interop_unsafe';
 import 'dart:typed_data';
 import 'dart:js';
 import 'dart:html';
 import 'dart:async';
+import 'dart:ui' as ui;
+import 'dart:ui_web' as ui_web;
 
 import 'package:flutter/foundation.dart';
 import 'package:flutter_hbb/common/widgets/login.dart';
 import 'package:flutter_hbb/models/state_model.dart';
+import 'package:flutter_hbb/models/web_video_frame_queue.dart';
 
 import 'package:flutter_hbb/web/bridge.dart';
 import 'package:flutter_hbb/common.dart';
@@ -18,6 +22,22 @@ import 'package:uuid/uuid.dart';
 final List<StreamSubscription<MouseEvent>> mouseListeners = [];
 final List<StreamSubscription<KeyboardEvent>> keyListeners = [];
 
+// WebCodecs VideoFrames handed over from js/src/webcodecs.js arrive as plain
+// interop objects (the package language version predates extension types).
+// This side owns each frame and must close it quickly: hardware decoders
+// stall once their small output frame pool is exhausted.
+int _videoFrameWidth(JSObject frame) =>
+    frame.getProperty<JSNumber>('displayWidth'.toJS).toDartInt;
+int _videoFrameHeight(JSObject frame) =>
+    frame.getProperty<JSNumber>('displayHeight'.toJS).toDartInt;
+void _closeVideoFrame(JSObject frame) {
+  try {
+    frame.callMethod<JSAny?>('close'.toJS);
+  } catch (error) {
+    debugPrint('VideoFrame.close failed: $error');
+  }
+}
+
 typedef HandleEvent = Future<void> Function(Map<String, dynamic> evt);
 
 class PlatformFFI {
@@ -33,6 +53,13 @@ class PlatformFFI {
   }
 
   PlatformFFI._() {
+    _videoFrameQueue = WebVideoFrameQueue(
+      importFrame: _importVideoFrame,
+      closeFrame: _closeVideoFrame,
+      disposeImage: (image) => image.dispose(),
+      onImportError: _handleVideoFrameImportError,
+      onCallbackError: _handleVideoImageCallbackError,
+    );
     window.document.addEventListener(
         'visibilitychange',
         (event) => {
@@ -162,6 +189,46 @@ class PlatformFFI {
     };
   }
 
+  late final WebVideoFrameQueue<JSObject, ui.Image> _videoFrameQueue;
+
+  // Zero-readback video path: the JS decoder hands decoded VideoFrames here
+  // (checking typeof window.onVideoFrame before every frame), and the engine
+  // imports them GPU-to-GPU via createImageBitmap. Unregistering the JS global
+  // reverts the JS side to the RGBA readback path.
+  void setVideoFrameCallback(
+      Future<void> Function(int, ui.Image, bool Function()) fun) {
+    _videoFrameQueue.beginSession(fun);
+    if (!_videoFrameQueue.isEnabled) return;
+    globalContext.setProperty(
+      'onVideoFrame'.toJS,
+      ((JSNumber display, JSObject frame) {
+        _videoFrameQueue.submit(display.toDartInt, frame);
+      }).toJS,
+    );
+  }
+
+  void clearVideoFrameCallback() {
+    _videoFrameQueue.endSession();
+    globalContext.setProperty('onVideoFrame'.toJS, null);
+  }
+
+  Future<ui.Image> _importVideoFrame(JSObject frame) async {
+    return await ui_web.createImageFromTextureSource(frame,
+        width: _videoFrameWidth(frame), height: _videoFrameHeight(frame));
+  }
+
+  void _handleVideoFrameImportError(Object error, StackTrace stackTrace) {
+    debugPrintStack(
+        label: 'createImageFromTextureSource failed, using RGBA path: $error',
+        stackTrace: stackTrace);
+    globalContext.setProperty('onVideoFrame'.toJS, null);
+  }
+
+  void _handleVideoImageCallbackError(Object error, StackTrace stackTrace) {
+    debugPrintStack(
+        label: 'video image callback error: $error', stackTrace: stackTrace);
+  }
+
   void startDesktopWebListener() {
     mouseListeners.add(
         window.document.onContextMenu.listen((evt) => evt.preventDefault()));

flutter/lib/models/web_video_frame_queue.dart

@@ -0,0 +1,133 @@
+import 'dart:async';
+
+typedef VideoFrameImporter<Frame, Image> = Future<Image> Function(Frame frame);
+typedef VideoFrameCloser<Frame> = void Function(Frame frame);
+typedef VideoImageDisposer<Image> = void Function(Image image);
+typedef VideoSessionValidator = bool Function();
+typedef VideoImageCallback<Image> = Future<void> Function(
+    int display, Image image, VideoSessionValidator isCurrentSession);
+typedef VideoQueueErrorCallback = void Function(
+    Object error, StackTrace stackTrace);
+
+class WebVideoFrameQueue<Frame, Image> {
+  WebVideoFrameQueue({
+    required VideoFrameImporter<Frame, Image> importFrame,
+    required VideoFrameCloser<Frame> closeFrame,
+    required VideoImageDisposer<Image> disposeImage,
+    required VideoQueueErrorCallback onImportError,
+    required VideoQueueErrorCallback onCallbackError,
+  })  : _importFrame = importFrame,
+        _closeFrame = closeFrame,
+        _disposeImage = disposeImage,
+        _onImportError = onImportError,
+        _onCallbackError = onCallbackError;
+
+  final VideoFrameImporter<Frame, Image> _importFrame;
+  final VideoFrameCloser<Frame> _closeFrame;
+  final VideoImageDisposer<Image> _disposeImage;
+  final VideoQueueErrorCallback _onImportError;
+  final VideoQueueErrorCallback _onCallbackError;
+  final Map<int, _QueuedFrame<Frame>> _pending = {};
+
+  VideoImageCallback<Image>? _callback;
+  int _generation = 0;
+  bool _processing = false;
+  bool _enabled = true;
+
+  bool get isEnabled => _enabled;
+
+  void beginSession(VideoImageCallback<Image> callback) {
+    _invalidateSession();
+    _enabled = true;
+    _callback = callback;
+  }
+
+  void endSession() {
+    _invalidateSession();
+    _callback = null;
+  }
+
+  void _invalidateSession() {
+    _generation++;
+    for (final queued in _pending.values) {
+      _closeFrame(queued.frame);
+    }
+    _pending.clear();
+  }
+
+  bool submit(int display, Frame frame) {
+    if (!_enabled || _callback == null) {
+      _closeFrame(frame);
+      return false;
+    }
+    final replaced = _pending.remove(display);
+    if (replaced != null) {
+      _closeFrame(replaced.frame);
+    }
+    _pending[display] = _QueuedFrame(display, frame, _generation);
+    _startProcessing();
+    return true;
+  }
+
+  void _startProcessing() {
+    if (_processing) return;
+    _processing = true;
+    unawaited(Future<void>(_process));
+  }
+
+  Future<void> _process() async {
+    while (_pending.isNotEmpty) {
+      final display = _pending.keys.first;
+      final queued = _pending.remove(display)!;
+      if (!_enabled || queued.generation != _generation) {
+        _closeFrame(queued.frame);
+        continue;
+      }
+      await _importAndDeliver(queued);
+    }
+    _processing = false;
+  }
+
+  Future<void> _importAndDeliver(_QueuedFrame<Frame> queued) async {
+    Image? image;
+    try {
+      image = await _importFrame(queued.frame);
+    } catch (error, stackTrace) {
+      if (queued.generation == _generation) {
+        _enabled = false;
+        _onImportError(error, stackTrace);
+      }
+    } finally {
+      _closeFrame(queued.frame);
+    }
+    if (image != null) {
+      await _deliver(queued, image);
+    }
+  }
+
+  Future<void> _deliver(_QueuedFrame<Frame> queued, Image image) async {
+    final callback = _callback;
+    bool isCurrentSession() =>
+        _enabled &&
+        queued.generation == _generation &&
+        identical(callback, _callback);
+    if (!isCurrentSession() || callback == null) {
+      _disposeImage(image);
+      return;
+    }
+    try {
+      await callback(queued.display, image, isCurrentSession);
+    } catch (error, stackTrace) {
+      _disposeImage(image);
+      _onCallbackError(error, stackTrace);
+    }
+  }
+}
+
+class _QueuedFrame<Frame> {
+  const _QueuedFrame(this.display, this.frame, this.generation);
+
+  final int display;
+  final Frame frame;
+  final int generation;
+}

flutter/lib/web/dummy.dart

@@ -12,3 +12,5 @@ Future<void> webSendLocalFiles(
     required bool isRemote}) {
   throw UnimplementedError("webSendLocalFiles");
 }
+
+Future<void> loadLocalTerminalFontIfNeeded() async {}

flutter/lib/web/terminal_font.dart

@@ -0,0 +1,33 @@
+import 'dart:html' as html;
+import 'dart:js' as js;
+import 'dart:typed_data';
+
+import 'package:flutter/foundation.dart';
+import 'package:flutter/services.dart';
+
+bool _loadRequested = false;
+
+/// When Google CDNs are unreachable, `index.html` sets
+/// `window.rustdeskLocalFonts` and `GoogleFonts.robotoMono()` cannot download
+/// the terminal font. Load the copy bundled with the web app instead,
+/// registered under the family name google_fonts gives the terminal's
+/// TextStyle ('RobotoMono_regular').
+Future<void> loadLocalTerminalFontIfNeeded() async {
+  if (_loadRequested || js.context['rustdeskLocalFonts'] != true) {
+    return;
+  }
+  _loadRequested = true;
+  try {
+    final req = await html.HttpRequest.request(
+      'fonts/RobotoMono-Regular.ttf',
+      responseType: 'arraybuffer',
+    );
+    final data = ByteData.view(req.response as ByteBuffer);
+    final loader = FontLoader('RobotoMono_regular')
+      ..addFont(Future.value(data));
+    await loader.load();
+  } catch (e) {
+    _loadRequested = false;
+    debugPrint('Failed to load bundled Roboto Mono: $e');
+  }
+}