From 4234b99029bf32c23098b4eaeec8efc135c8e80a Mon Sep 17 00:00:00 2001 From: RustDesk <71636191+rustdesk@users.noreply.github.com> Date: Fri, 7 Aug 2026 15:20:57 +0800 Subject: [PATCH] WebClient: 3.44 webcodecs offline (#15722) * feat(web): zero-readback WebCodecs video path Decoded VideoFrames from js/src/webcodecs.js are handed to Flutter via window.onVideoFrame and imported GPU-side with createImageFromTextureSource; any failure unregisters the hook so the JS side falls back to RGBA readback. Co-Authored-By: Claude Fable 5 * fix(web): load bundled terminal font when Google CDNs are unreachable In air-gapped deployments GoogleFonts.robotoMono() cannot download the terminal font; when index.html signals offline mode, load the copy bundled with the web app under the family name google_fonts registers. Part of the fix for rustdesk/rustdesk-server-pro#996. Co-Authored-By: Claude Fable 5 * ci: bump windows arm64 to Flutter 3.44.8, add web build patch script apply_flutter_3.44_web_patches.sh prepares a 3.44.x web build on top of the shared source patches: qr_code_scanner's web impl needs dart:ui_web for the removed platformViewRegistry, and flutter/web/fonts is refreshed to the font paths the 3.44 engine requests. The disabled build-rustdesk-web job runs it automatically once FLUTTER_VERSION moves to 3.44.x, and version-guarded 'Patch flutter' steps no longer fail when the guard does not match. Co-Authored-By: Claude Fable 5 * fix(web): prevent stale WebCodecs frames across sessions Signed-off-by: fufesou * fix(web): harden WebCodecs reconnect and Flutter 3.44 patches Signed-off-by: fufesou * fix(ci): harden Flutter 3.44 patch input validation Validate required files before checking patch state, parameterize the theme-range validator, and prevent missing inputs from satisfying NO_MATCHES checks. Signed-off-by: fufesou * Remove unused code Signed-off-by: fufesou * fix(web): retry font loading and dispose stale decoded images Signed-off-by: fufesou * remove unused code Signed-off-by: fufesou * fix(web): Bad state: RenderBox was not laid out Signed-off-by: fufesou --------- Signed-off-by: fufesou Co-authored-by: Claude Fable 5 Co-authored-by: fufesou --- .../apply_flutter_3.44_source_patches.sh | 115 ++++++++++++++- .../patches/apply_flutter_3.44_web_patches.sh | 51 +++++++ .github/workflows/bridge.yml | 2 +- .github/workflows/flutter-build.yml | 35 ++++- flutter/lib/main.dart | 3 +- flutter/lib/mobile/pages/terminal_page.dart | 6 + flutter/lib/models/model.dart | 37 ++++- flutter/lib/models/native_model.dart | 7 + flutter/lib/models/web_model.dart | 67 +++++++++ flutter/lib/models/web_video_frame_queue.dart | 133 ++++++++++++++++++ flutter/lib/web/dummy.dart | 2 + flutter/lib/web/terminal_font.dart | 33 +++++ 12 files changed, 472 insertions(+), 19 deletions(-) create mode 100755 .github/patches/apply_flutter_3.44_web_patches.sh create mode 100644 flutter/lib/models/web_video_frame_queue.dart create mode 100644 flutter/lib/web/terminal_font.dart diff --git a/.github/patches/apply_flutter_3.44_source_patches.sh b/.github/patches/apply_flutter_3.44_source_patches.sh index 3a7ab99dc..2b4bfcc0d 100644 --- a/.github/patches/apply_flutter_3.44_source_patches.sh +++ b/.github/patches/apply_flutter_3.44_source_patches.sh @@ -17,6 +17,109 @@ # therefore CRLF-safe. set -euo pipefail +readonly NO_MATCHES=0 +readonly SINGLE_MATCH=1 +readonly THEME_MATCHES=2 + +has_exact_count() { + local -r expected_count="$1" + local -r pattern="$2" + local -r file="$3" + local actual_count + [[ -r "$file" ]] || return 1 + actual_count="$(grep -cF "$pattern" "$file" || true)" + [[ "$actual_count" -eq "$expected_count" ]] +} + +# The target background-color line must directly follow DialogThemeData in the selected range. +has_dialog_background_in_theme_range() { + local -r start_pattern="$1" + local -r end_pattern="$2" + local -r target_pattern="$3" + local -r file="$4" + awk -v start_pattern="$start_pattern" \ + -v end_pattern="$end_pattern" \ + -v target_pattern="$target_pattern" ' + index($0, start_pattern) { + in_theme = 1 + next + } + in_theme && index($0, end_pattern) { + exit + } + in_theme && index($0, "dialogTheme: DialogThemeData(") { + if (getline > 0) { + line = $0 + sub(/\r$/, "", line) + sub(/^[[:space:]]+/, "", line) + matched = line == target_pattern + } + exit + } + END { + exit matched ? 0 : 1 + } + ' "$file" +} + +validate_patch_inputs() { + if [[ ! -f flutter/lib/common.dart || ! -r flutter/lib/common.dart ]]; then + echo "Flutter 3.44 source patch input is missing or unreadable: flutter/lib/common.dart" >&2 + return 1 + fi + if [[ ! -f flutter/pubspec.yaml || ! -r flutter/pubspec.yaml ]]; then + echo "Flutter 3.44 source patch input is missing or unreadable: flutter/pubspec.yaml" >&2 + return 1 + fi +} + +is_complete_patch_state() { + has_exact_count "$THEME_MATCHES" 'dialogTheme: DialogThemeData(' flutter/lib/common.dart && + has_exact_count "$THEME_MATCHES" 'tabBarTheme: const TabBarThemeData(' flutter/lib/common.dart && + has_exact_count "$SINGLE_MATCH" 'backgroundColor: Colors.white,' flutter/lib/common.dart && + has_exact_count "$SINGLE_MATCH" 'backgroundColor: Color(0xFF18191E),' flutter/lib/common.dart && + has_exact_count "$SINGLE_MATCH" 'extended_text: 15.0.2' flutter/pubspec.yaml && + has_exact_count "$SINGLE_MATCH" 'google_fonts: ^8.1.0' flutter/pubspec.yaml && + has_exact_count "$NO_MATCHES" 'dialogTheme: DialogTheme(' flutter/lib/common.dart && + has_exact_count "$NO_MATCHES" 'tabBarTheme: const TabBarTheme(' flutter/lib/common.dart && + has_exact_count "$NO_MATCHES" 'extended_text: 14.0.0' flutter/pubspec.yaml && + has_exact_count "$NO_MATCHES" 'google_fonts: ^6.2.1' flutter/pubspec.yaml && + has_dialog_background_in_theme_range 'static ThemeData lightTheme = ThemeData(' \ + 'static ThemeData darkTheme = ThemeData(' 'backgroundColor: Colors.white,' \ + flutter/lib/common.dart && + has_dialog_background_in_theme_range 'static ThemeData darkTheme = ThemeData(' \ + 'scrollbarTheme: scrollbarThemeDark,' 'backgroundColor: Color(0xFF18191E),' \ + flutter/lib/common.dart +} + +is_unpatched_state() { + has_exact_count "$THEME_MATCHES" 'dialogTheme: DialogTheme(' flutter/lib/common.dart && + has_exact_count "$THEME_MATCHES" 'tabBarTheme: const TabBarTheme(' flutter/lib/common.dart && + has_exact_count "$SINGLE_MATCH" 'extended_text: 14.0.0' flutter/pubspec.yaml && + has_exact_count "$SINGLE_MATCH" 'google_fonts: ^6.2.1' flutter/pubspec.yaml && + has_exact_count "$NO_MATCHES" 'dialogTheme: DialogThemeData(' flutter/lib/common.dart && + has_exact_count "$NO_MATCHES" 'tabBarTheme: const TabBarThemeData(' flutter/lib/common.dart && + has_exact_count "$NO_MATCHES" 'backgroundColor: Colors.white,' flutter/lib/common.dart && + has_exact_count "$NO_MATCHES" 'backgroundColor: Color(0xFF18191E),' flutter/lib/common.dart && + has_exact_count "$NO_MATCHES" 'extended_text: 15.0.2' flutter/pubspec.yaml && + has_exact_count "$NO_MATCHES" 'google_fonts: ^8.1.0' flutter/pubspec.yaml +} + +if ! validate_patch_inputs; then + exit 1 +fi + +if is_complete_patch_state; then + echo "Flutter 3.44 source patches already applied." + git --no-pager diff -- flutter/lib/common.dart flutter/pubspec.yaml + exit 0 +fi + +if ! is_unpatched_state; then + echo "Flutter 3.44 source patches are partially applied or their anchors have drifted." >&2 + exit 1 +fi + # ThemeData API renames (Flutter 3.27+): sed -i 's/dialogTheme: DialogTheme(/dialogTheme: DialogThemeData(/g' flutter/lib/common.dart sed -i 's/tabBarTheme: const TabBarTheme(/tabBarTheme: const TabBarThemeData(/g' flutter/lib/common.dart @@ -28,12 +131,10 @@ sed -i '/static ThemeData darkTheme = ThemeData(/,/scrollbarTheme: scrollbarThem sed -i 's/extended_text: 14.0.0/extended_text: 15.0.2/' flutter/pubspec.yaml sed -i 's/google_fonts: \^6.2.1/google_fonts: ^8.1.0/' flutter/pubspec.yaml -# Fail loudly if any expected string drifted, so we never silently build unpatched: -grep -qF 'dialogTheme: DialogThemeData(' flutter/lib/common.dart -grep -qF 'tabBarTheme: const TabBarThemeData(' flutter/lib/common.dart -grep -qF 'backgroundColor: Colors.white,' flutter/lib/common.dart -grep -qF 'backgroundColor: Color(0xFF18191E),' flutter/lib/common.dart -grep -qF 'extended_text: 15.0.2' flutter/pubspec.yaml -grep -qF 'google_fonts: ^8.1.0' flutter/pubspec.yaml +# Fail loudly if any expected substitution did not produce the complete state. +if ! is_complete_patch_state; then + echo "Flutter 3.44 source patches did not produce the expected state." >&2 + exit 1 +fi git --no-pager diff -- flutter/lib/common.dart flutter/pubspec.yaml diff --git a/.github/patches/apply_flutter_3.44_web_patches.sh b/.github/patches/apply_flutter_3.44_web_patches.sh new file mode 100755 index 000000000..24ce7f1b4 --- /dev/null +++ b/.github/patches/apply_flutter_3.44_web_patches.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Prepares a web build on Flutter 3.44.x. Companion to +# apply_flutter_3.44_source_patches.sh (which it runs first): the web target +# additionally needs qr_code_scanner's web implementation patched for the +# dart:ui platformViewRegistry removal, and flutter/web/fonts refreshed with +# the font paths the 3.44 engine requests for offline/air-gapped support +# (rustdesk-server-pro#996; see flutter/web/fonts/sync_fonts.py). +# +# Run from the repository root with Flutter 3.44.x on PATH, then build: +# bash .github/patches/apply_flutter_3.44_web_patches.sh +# (cd flutter && flutter build web --release) # or ./web/js/flutter_build.py +# +# Idempotent. To undo the source changes locally: +# git checkout -- flutter/lib/common.dart flutter/pubspec.yaml flutter/pubspec.lock +set -euo pipefail + +flutter --version | grep -q "Flutter 3\.44\." || { + echo "Flutter 3.44.x must be on PATH; found:" >&2 + flutter --version | grep "^Flutter" >&2 || true + exit 1 +} + +# Shared 3.44 source/pubspec patches own their complete-state validation. +bash .github/patches/apply_flutter_3.44_source_patches.sh + +# Populate the pub cache with the 3.44 dependency resolution. +(cd flutter && flutter pub get) + +# qr_code_scanner 1.0.1 (unmaintained) reads platformViewRegistry from +# dart:ui, which Flutter 3.44 removed; point it at dart:ui_web instead. The +# patched file also compiles on Flutter 3.24 (dart:ui_web exists there), so +# mutating the shared pub cache is safe for other local builds. +QR_WEB="${PUB_CACHE:-$HOME/.pub-cache}/hosted/pub.dev/qr_code_scanner-1.0.1/lib/src/web/flutter_qr_web.dart" +if ! grep -qF "dart:ui_web" "$QR_WEB"; then + sed -i.bak "s|import 'dart:ui' as ui;|import 'dart:ui' as ui; import 'dart:ui_web' as ui_web;|" "$QR_WEB" + rm -f "$QR_WEB.bak" +fi +if grep -qF "ui.platformViewRegistry" "$QR_WEB"; then + sed -i.bak "s|ui\.platformViewRegistry|ui_web.platformViewRegistry|g" "$QR_WEB" + rm -f "$QR_WEB.bak" +fi + +# Mirror the fonts this engine version requests into flutter/web/fonts. +python3 flutter/web/fonts/sync_fonts.py + +# Fail loudly if any expected state is missing: +grep -qF "import 'dart:ui' as ui; import 'dart:ui_web' as ui_web;" "$QR_WEB" +grep -qF "ui_web.platformViewRegistry" "$QR_WEB" +grep -qF 'google_fonts: ^8.1.0' flutter/pubspec.yaml + +echo "Flutter 3.44 web patches applied." diff --git a/.github/workflows/bridge.yml b/.github/workflows/bridge.yml index a7b74fa55..9d31399c8 100644 --- a/.github/workflows/bridge.yml +++ b/.github/workflows/bridge.yml @@ -30,7 +30,7 @@ jobs: target: x86_64-unknown-linux-gnu, os: ubuntu-22.04, extra-build-args: "", - flutter-version: "3.44.0", + flutter-version: "3.44.8", artifact-name: "bridge-artifact-flutter-3.44", } steps: diff --git a/.github/workflows/flutter-build.yml b/.github/workflows/flutter-build.yml index 95cfdd8e3..4f1dbb1c9 100644 --- a/.github/workflows/flutter-build.yml +++ b/.github/workflows/flutter-build.yml @@ -31,7 +31,7 @@ env: # engine is 3.44. Every other platform stays on FLUTTER_VERSION (3.24.5) until Windows 7 # support is restored after the upstream-wide Flutter bump. The arm64 job patches the few # 3.44-only source/pubspec changes on the fly (see "Patch RustDesk sources for Flutter 3.44"). - FLUTTER_WINDOWS_ARM_VERSION: "3.44.0" + FLUTTER_WINDOWS_ARM_VERSION: "3.44.8" # for arm64 linux because official Dart SDK does not work FLUTTER_ELINUX_VERSION: "3.16.9" TAG_NAME: "${{ inputs.upload-tag }}" @@ -224,7 +224,9 @@ jobs: run: | cp .github/patches/flutter_3.24.4_dropdown_menu_enableFilter.diff $(dirname $(dirname $(which flutter))) cd $(dirname $(dirname $(which flutter))) - [[ "3.24.5" == ${{env.FLUTTER_VERSION}} ]] && git apply flutter_3.24.4_dropdown_menu_enableFilter.diff + if [[ "3.24.5" == ${{env.FLUTTER_VERSION}} ]]; then + git apply flutter_3.24.4_dropdown_menu_enableFilter.diff + fi - name: Patch RustDesk sources for Flutter 3.44 (arm64) # arm64 is the only target on Flutter 3.44; apply its source/pubspec deltas on the fly @@ -595,7 +597,9 @@ jobs: - name: Patch flutter run: | cd $(dirname $(dirname $(which flutter))) - [[ "3.24.5" == ${{env.FLUTTER_VERSION}} ]] && git apply ${{ github.workspace }}/.github/patches/flutter_3.24.4_dropdown_menu_enableFilter.diff + if [[ "3.24.5" == ${{env.FLUTTER_VERSION}} ]]; then + git apply ${{ github.workspace }}/.github/patches/flutter_3.24.4_dropdown_menu_enableFilter.diff + fi - name: Setup vcpkg with Github Actions binary cache uses: lukka/run-vcpkg@b1a0dd252f06b9e25b3c022a9a03bd7a427fb6a2 # v11 @@ -774,7 +778,9 @@ jobs: - name: Patch flutter run: | cd $(dirname $(dirname $(which flutter))) - [[ "3.24.5" == ${{env.FLUTTER_VERSION}} ]] && git apply ${{ github.workspace }}/.github/patches/flutter_3.24.4_dropdown_menu_enableFilter.diff + if [[ "3.24.5" == ${{env.FLUTTER_VERSION}} ]]; then + git apply ${{ github.workspace }}/.github/patches/flutter_3.24.4_dropdown_menu_enableFilter.diff + fi - name: Workaround for flutter issue shell: bash @@ -1033,7 +1039,9 @@ jobs: - name: Patch flutter run: | cd $(dirname $(dirname $(which flutter))) - [[ "3.24.5" == ${{env.ANDROID_FLUTTER_VERSION}} ]] && git apply ${{ github.workspace }}/.github/patches/flutter_3.24.4_dropdown_menu_enableFilter.diff + if [[ "3.24.5" == ${{env.ANDROID_FLUTTER_VERSION}} ]]; then + git apply ${{ github.workspace }}/.github/patches/flutter_3.24.4_dropdown_menu_enableFilter.diff + fi - uses: nttld/setup-ndk@ed92fe6cadad69be94a966a7ee3271275e62f779 # v1 id: setup-ndk @@ -1305,7 +1313,9 @@ jobs: - name: Patch flutter run: | cd $(dirname $(dirname $(which flutter))) - [[ "3.24.5" == ${{env.ANDROID_FLUTTER_VERSION}} ]] && git apply ${{ github.workspace }}/.github/patches/flutter_3.24.4_dropdown_menu_enableFilter.diff + if [[ "3.24.5" == ${{env.ANDROID_FLUTTER_VERSION}} ]]; then + git apply ${{ github.workspace }}/.github/patches/flutter_3.24.4_dropdown_menu_enableFilter.diff + fi - name: Restore bridge files uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -2413,7 +2423,18 @@ jobs: shell: bash run: | cd $(dirname $(dirname $(which flutter))) - [[ "3.24.5" == ${{env.FLUTTER_VERSION}} ]] && git apply ${{ github.workspace }}/.github/patches/flutter_3.24.4_dropdown_menu_enableFilter.diff + if [[ "3.24.5" == ${{env.FLUTTER_VERSION}} ]]; then + git apply ${{ github.workspace }}/.github/patches/flutter_3.24.4_dropdown_menu_enableFilter.diff + fi + + - name: Patch sources for Flutter 3.44 web + # No-op while the web stays on Flutter 3.24.5; makes this job work as-is + # once FLUTTER_VERSION moves to 3.44.x (qr_code_scanner + fonts, see script). + shell: bash + run: | + if [[ "${{ env.FLUTTER_VERSION }}" == 3.44.* ]]; then + bash .github/patches/apply_flutter_3.44_web_patches.sh + fi # https://rustdesk.com/docs/en/dev/build/web/ - name: Build web diff --git a/flutter/lib/main.dart b/flutter/lib/main.dart index 9bd68ed60..7e0a8cb2b 100644 --- a/flutter/lib/main.dart +++ b/flutter/lib/main.dart @@ -588,7 +588,8 @@ _registerEventHandler() { Widget keyListenerBuilder(BuildContext context, Widget? child) { return RawKeyboardListener( - focusNode: FocusNode(), + // `skipTraversal: isWeb` is to fix "Bad state: RenderBox was not laid out: minified:aeL#c19e4" + focusNode: FocusNode(skipTraversal: isWeb), child: child ?? Container(), onKey: (RawKeyEvent event) { if (event.logicalKey == LogicalKeyboardKey.shiftLeft) { diff --git a/flutter/lib/mobile/pages/terminal_page.dart b/flutter/lib/mobile/pages/terminal_page.dart index a4a76f9af..800b0f8f4 100644 --- a/flutter/lib/mobile/pages/terminal_page.dart +++ b/flutter/lib/mobile/pages/terminal_page.dart @@ -10,6 +10,8 @@ import 'package:flutter_hbb/models/model.dart'; import 'package:flutter_hbb/models/platform_model.dart'; import 'package:flutter_hbb/models/terminal_model.dart'; import 'package:flutter_hbb/mobile/terminal_keyboard_utils.dart'; +import 'package:flutter_hbb/web/dummy.dart' + if (dart.library.html) 'package:flutter_hbb/web/terminal_font.dart'; import 'package:google_fonts/google_fonts.dart'; import 'package:xterm/xterm.dart'; import '../../desktop/pages/terminal_connection_manager.dart'; @@ -67,6 +69,10 @@ class _TerminalPageState extends State super.initState(); WidgetsBinding.instance.addObserver(this); + if (isWeb) { + loadLocalTerminalFontIfNeeded(); + } + debugPrint( '[TerminalPage] Initializing terminal ${widget.terminalId} for peer ${widget.id}'); diff --git a/flutter/lib/models/model.dart b/flutter/lib/models/model.dart index 175e3ff2d..4a6088bd3 100644 --- a/flutter/lib/models/model.dart +++ b/flutter/lib/models/model.dart @@ -1952,6 +1952,12 @@ class ImageModel with ChangeNotifier { platformFFI.nextRgba(sessionId, display); } + // web only: image already created from a decoded WebCodecs frame + Future onImage( + int display, ui.Image image, bool Function() isCurrentSession) async { + await update(image, isCurrentSession: isCurrentSession); + } + decodeAndUpdate(int display, Uint8List rgba) async { final pid = parent.target?.id; final rect = parent.target?.ffiModel.pi.getDisplayRect(display); @@ -1963,11 +1969,16 @@ class ImageModel with ChangeNotifier { ? ui.PixelFormat.rgba8888 : ui.PixelFormat.bgra8888, ); - if (parent.target?.id != pid) return; + if (parent.target?.id != pid) { + image?.dispose(); + return; + } await update(image); } - update(ui.Image? image) async { + Future update(ui.Image? image, + {bool Function()? isCurrentSession}) async { + if (_disposeIfStale(image, isCurrentSession)) return; if (_image == null && image != null) { if (isDesktop || isWebDesktop) { await parent.target?.canvasModel.updateViewStyle(); @@ -1978,11 +1989,19 @@ class ImageModel with ChangeNotifier { await initializeCursorAndCanvas(parent.target!); } } + if (_disposeIfStale(image, isCurrentSession)) return; _image?.dispose(); _image = image; if (image != null) notifyListeners(); } + bool _disposeIfStale(ui.Image? image, bool Function()? isCurrentSession) { + if (image == null || isCurrentSession == null) return false; + if (isCurrentSession()) return false; + image.dispose(); + return true; + } + // mobile only double get maxScale { if (_image == null) return 1.5; @@ -3853,6 +3872,15 @@ class FFI { onEvent2UIRgba(); imageModel.onRgba(display, data); }); + platformFFI.setVideoFrameCallback((int display, ui.Image image, + bool Function() isCurrentSession) async { + if (!isCurrentSession()) { + image.dispose(); + return; + } + await onEvent2UIRgba(); + await imageModel.onImage(display, image, isCurrentSession); + }); this.id = id; return; } @@ -3940,7 +3968,7 @@ class FFI { this.id = id; } - void onEvent2UIRgba() async { + Future onEvent2UIRgba() async { if (ffiModel.waitForImageDialogShow.isTrue) { ffiModel.waitForImageDialogShow.value = false; ffiModel.waitForImageTimer?.cancel(); @@ -3996,6 +4024,9 @@ class FFI { /// Close the remote session. Future close({bool closeSession = true}) async { closed = true; + if (isWeb) { + platformFFI.clearVideoFrameCallback(); + } chatModel.close(); // Close all terminal models for (final model in _terminalModels.values) { diff --git a/flutter/lib/models/native_model.dart b/flutter/lib/models/native_model.dart index e73cbc0cb..8c3c5cf71 100644 --- a/flutter/lib/models/native_model.dart +++ b/flutter/lib/models/native_model.dart @@ -1,6 +1,7 @@ import 'dart:convert'; import 'dart:ffi'; import 'dart:io'; +import 'dart:ui' as ui; import 'package:device_info_plus/device_info_plus.dart'; import 'package:external_path/external_path.dart'; @@ -283,6 +284,12 @@ class PlatformFFI { void setRgbaCallback(void Function(int, Uint8List) fun) async {} + // web only, decoded WebCodecs frames arriving as ready-made images + void setVideoFrameCallback( + Future Function(int, ui.Image, bool Function()) fun) {} + + void clearVideoFrameCallback() {} + void startDesktopWebListener() {} void stopDesktopWebListener() {} diff --git a/flutter/lib/models/web_model.dart b/flutter/lib/models/web_model.dart index 5241c3974..b65825e51 100644 --- a/flutter/lib/models/web_model.dart +++ b/flutter/lib/models/web_model.dart @@ -2,14 +2,18 @@ import 'dart:convert'; import 'dart:js_interop'; +import 'dart:js_interop_unsafe'; import 'dart:typed_data'; import 'dart:js'; import 'dart:html'; import 'dart:async'; +import 'dart:ui' as ui; +import 'dart:ui_web' as ui_web; import 'package:flutter/foundation.dart'; import 'package:flutter_hbb/common/widgets/login.dart'; import 'package:flutter_hbb/models/state_model.dart'; +import 'package:flutter_hbb/models/web_video_frame_queue.dart'; import 'package:flutter_hbb/web/bridge.dart'; import 'package:flutter_hbb/common.dart'; @@ -18,6 +22,22 @@ import 'package:uuid/uuid.dart'; final List> mouseListeners = []; final List> keyListeners = []; +// WebCodecs VideoFrames handed over from js/src/webcodecs.js arrive as plain +// interop objects (the package language version predates extension types). +// This side owns each frame and must close it quickly: hardware decoders +// stall once their small output frame pool is exhausted. +int _videoFrameWidth(JSObject frame) => + frame.getProperty('displayWidth'.toJS).toDartInt; +int _videoFrameHeight(JSObject frame) => + frame.getProperty('displayHeight'.toJS).toDartInt; +void _closeVideoFrame(JSObject frame) { + try { + frame.callMethod('close'.toJS); + } catch (error) { + debugPrint('VideoFrame.close failed: $error'); + } +} + typedef HandleEvent = Future Function(Map evt); class PlatformFFI { @@ -33,6 +53,13 @@ class PlatformFFI { } PlatformFFI._() { + _videoFrameQueue = WebVideoFrameQueue( + importFrame: _importVideoFrame, + closeFrame: _closeVideoFrame, + disposeImage: (image) => image.dispose(), + onImportError: _handleVideoFrameImportError, + onCallbackError: _handleVideoImageCallbackError, + ); window.document.addEventListener( 'visibilitychange', (event) => { @@ -162,6 +189,46 @@ class PlatformFFI { }; } + late final WebVideoFrameQueue _videoFrameQueue; + + // Zero-readback video path: the JS decoder hands decoded VideoFrames here + // (checking typeof window.onVideoFrame before every frame), and the engine + // imports them GPU-to-GPU via createImageBitmap. Unregistering the JS global + // reverts the JS side to the RGBA readback path. + void setVideoFrameCallback( + Future Function(int, ui.Image, bool Function()) fun) { + _videoFrameQueue.beginSession(fun); + if (!_videoFrameQueue.isEnabled) return; + globalContext.setProperty( + 'onVideoFrame'.toJS, + ((JSNumber display, JSObject frame) { + _videoFrameQueue.submit(display.toDartInt, frame); + }).toJS, + ); + } + + void clearVideoFrameCallback() { + _videoFrameQueue.endSession(); + globalContext.setProperty('onVideoFrame'.toJS, null); + } + + Future _importVideoFrame(JSObject frame) async { + return await ui_web.createImageFromTextureSource(frame, + width: _videoFrameWidth(frame), height: _videoFrameHeight(frame)); + } + + void _handleVideoFrameImportError(Object error, StackTrace stackTrace) { + debugPrintStack( + label: 'createImageFromTextureSource failed, using RGBA path: $error', + stackTrace: stackTrace); + globalContext.setProperty('onVideoFrame'.toJS, null); + } + + void _handleVideoImageCallbackError(Object error, StackTrace stackTrace) { + debugPrintStack( + label: 'video image callback error: $error', stackTrace: stackTrace); + } + void startDesktopWebListener() { mouseListeners.add( window.document.onContextMenu.listen((evt) => evt.preventDefault())); diff --git a/flutter/lib/models/web_video_frame_queue.dart b/flutter/lib/models/web_video_frame_queue.dart new file mode 100644 index 000000000..b78b69166 --- /dev/null +++ b/flutter/lib/models/web_video_frame_queue.dart @@ -0,0 +1,133 @@ +import 'dart:async'; + +typedef VideoFrameImporter = Future Function(Frame frame); +typedef VideoFrameCloser = void Function(Frame frame); +typedef VideoImageDisposer = void Function(Image image); +typedef VideoSessionValidator = bool Function(); +typedef VideoImageCallback = Future Function( + int display, Image image, VideoSessionValidator isCurrentSession); +typedef VideoQueueErrorCallback = void Function( + Object error, StackTrace stackTrace); + +class WebVideoFrameQueue { + WebVideoFrameQueue({ + required VideoFrameImporter importFrame, + required VideoFrameCloser closeFrame, + required VideoImageDisposer disposeImage, + required VideoQueueErrorCallback onImportError, + required VideoQueueErrorCallback onCallbackError, + }) : _importFrame = importFrame, + _closeFrame = closeFrame, + _disposeImage = disposeImage, + _onImportError = onImportError, + _onCallbackError = onCallbackError; + + final VideoFrameImporter _importFrame; + final VideoFrameCloser _closeFrame; + final VideoImageDisposer _disposeImage; + final VideoQueueErrorCallback _onImportError; + final VideoQueueErrorCallback _onCallbackError; + final Map> _pending = {}; + + VideoImageCallback? _callback; + int _generation = 0; + bool _processing = false; + bool _enabled = true; + + bool get isEnabled => _enabled; + + void beginSession(VideoImageCallback callback) { + _invalidateSession(); + _enabled = true; + _callback = callback; + } + + void endSession() { + _invalidateSession(); + _callback = null; + } + + void _invalidateSession() { + _generation++; + for (final queued in _pending.values) { + _closeFrame(queued.frame); + } + _pending.clear(); + } + + bool submit(int display, Frame frame) { + if (!_enabled || _callback == null) { + _closeFrame(frame); + return false; + } + final replaced = _pending.remove(display); + if (replaced != null) { + _closeFrame(replaced.frame); + } + _pending[display] = _QueuedFrame(display, frame, _generation); + _startProcessing(); + return true; + } + + void _startProcessing() { + if (_processing) return; + _processing = true; + unawaited(Future(_process)); + } + + Future _process() async { + while (_pending.isNotEmpty) { + final display = _pending.keys.first; + final queued = _pending.remove(display)!; + if (!_enabled || queued.generation != _generation) { + _closeFrame(queued.frame); + continue; + } + await _importAndDeliver(queued); + } + _processing = false; + } + + Future _importAndDeliver(_QueuedFrame queued) async { + Image? image; + try { + image = await _importFrame(queued.frame); + } catch (error, stackTrace) { + if (queued.generation == _generation) { + _enabled = false; + _onImportError(error, stackTrace); + } + } finally { + _closeFrame(queued.frame); + } + if (image != null) { + await _deliver(queued, image); + } + } + + Future _deliver(_QueuedFrame queued, Image image) async { + final callback = _callback; + bool isCurrentSession() => + _enabled && + queued.generation == _generation && + identical(callback, _callback); + if (!isCurrentSession() || callback == null) { + _disposeImage(image); + return; + } + try { + await callback(queued.display, image, isCurrentSession); + } catch (error, stackTrace) { + _disposeImage(image); + _onCallbackError(error, stackTrace); + } + } +} + +class _QueuedFrame { + const _QueuedFrame(this.display, this.frame, this.generation); + + final int display; + final Frame frame; + final int generation; +} diff --git a/flutter/lib/web/dummy.dart b/flutter/lib/web/dummy.dart index b9e3b80b6..0063c38bd 100644 --- a/flutter/lib/web/dummy.dart +++ b/flutter/lib/web/dummy.dart @@ -12,3 +12,5 @@ Future webSendLocalFiles( required bool isRemote}) { throw UnimplementedError("webSendLocalFiles"); } + +Future loadLocalTerminalFontIfNeeded() async {} diff --git a/flutter/lib/web/terminal_font.dart b/flutter/lib/web/terminal_font.dart new file mode 100644 index 000000000..964924e4c --- /dev/null +++ b/flutter/lib/web/terminal_font.dart @@ -0,0 +1,33 @@ +import 'dart:html' as html; +import 'dart:js' as js; +import 'dart:typed_data'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/services.dart'; + +bool _loadRequested = false; + +/// When Google CDNs are unreachable, `index.html` sets +/// `window.rustdeskLocalFonts` and `GoogleFonts.robotoMono()` cannot download +/// the terminal font. Load the copy bundled with the web app instead, +/// registered under the family name google_fonts gives the terminal's +/// TextStyle ('RobotoMono_regular'). +Future loadLocalTerminalFontIfNeeded() async { + if (_loadRequested || js.context['rustdeskLocalFonts'] != true) { + return; + } + _loadRequested = true; + try { + final req = await html.HttpRequest.request( + 'fonts/RobotoMono-Regular.ttf', + responseType: 'arraybuffer', + ); + final data = ByteData.view(req.response as ByteBuffer); + final loader = FontLoader('RobotoMono_regular') + ..addFont(Future.value(data)); + await loader.load(); + } catch (e) { + _loadRequested = false; + debugPrint('Failed to load bundled Roboto Mono: $e'); + } +}