diff --git a/flutter/lib/common.dart b/flutter/lib/common.dart index ff8dfbb09..6d3e4c3b7 100644 --- a/flutter/lib/common.dart +++ b/flutter/lib/common.dart @@ -19,6 +19,7 @@ import 'package:flutter_hbb/utils/multi_window_manager.dart'; import 'package:flutter_hbb/utils/platform_channel.dart'; import 'package:flutter_svg/flutter_svg.dart'; import 'package:get/get.dart'; +import 'package:texture_rgba_renderer/texture_rgba_renderer.dart'; import 'package:uni_links/uni_links.dart'; import 'package:uni_links_desktop/uni_links_desktop.dart'; import 'package:url_launcher/url_launcher.dart'; @@ -44,6 +45,10 @@ var isWeb = false; var isWebDesktop = false; var version = ""; int androidVersion = 0; +/// Incriment count for textureId. +int _textureId = 0; +int get newTextureId => _textureId ++; +final textureRenderer = TextureRgbaRenderer(); /// only available for Windows target int windowsBuildNumber = 0; diff --git a/flutter/lib/desktop/pages/remote_page.dart b/flutter/lib/desktop/pages/remote_page.dart index f9db985d9..df9874172 100644 --- a/flutter/lib/desktop/pages/remote_page.dart +++ b/flutter/lib/desktop/pages/remote_page.dart @@ -63,6 +63,8 @@ class _RemotePageState extends State late RxBool _zoomCursor; late RxBool _remoteCursorMoved; late RxBool _keyboardEnabled; + late RxInt _textureId; + late int _textureKey; final _blockableOverlayState = BlockableOverlayState(); @@ -85,6 +87,8 @@ class _RemotePageState extends State _showRemoteCursor = ShowRemoteCursorState.find(id); _keyboardEnabled = KeyboardEnabledState.find(id); _remoteCursorMoved = RemoteCursorMovedState.find(id); + _textureKey = newTextureId; + _textureId = RxInt(-1); } void _removeStates(String id) { @@ -119,6 +123,16 @@ class _RemotePageState extends State if (!Platform.isLinux) { Wakelock.enable(); } + // Register texture. + _textureId.value = -1; + textureRenderer.createTexture(_textureKey).then((id) async { + if (id != -1) { + final ptr = await textureRenderer.getTexturePtr(_textureKey); + debugPrint("id: $id, texture_key: $_textureKey"); + platformFFI.registerTexture(widget.id, ptr); + _textureId.value = id; + } + }); _ffi.ffiModel.updateEventListener(widget.id); _ffi.qualityMonitorModel.checkShowQualityMonitor(widget.id); // Session option should be set after models.dart/FFI.start @@ -198,6 +212,7 @@ class _RemotePageState extends State Wakelock.disable(); } Get.delete(tag: widget.id); + textureRenderer.closeTexture(_textureKey); super.dispose(); _removeStates(widget.id); } @@ -346,6 +361,7 @@ class _RemotePageState extends State cursorOverImage: _cursorOverImage, keyboardEnabled: _keyboardEnabled, remoteCursorMoved: _remoteCursorMoved, + textureId: _textureId, listenerBuilder: (child) => _buildRawPointerMouseRegion(child, enterView, leaveView), ); @@ -383,6 +399,7 @@ class ImagePaint extends StatefulWidget { final RxBool cursorOverImage; final RxBool keyboardEnabled; final RxBool remoteCursorMoved; + final RxInt textureId; final Widget Function(Widget)? listenerBuilder; ImagePaint( @@ -392,6 +409,7 @@ class ImagePaint extends StatefulWidget { required this.cursorOverImage, required this.keyboardEnabled, required this.remoteCursorMoved, + required this.textureId, this.listenerBuilder}) : super(key: key); @@ -466,9 +484,15 @@ class _ImagePaintState extends State { final imageWidth = c.getDisplayWidth() * s; final imageHeight = c.getDisplayHeight() * s; final imageSize = Size(imageWidth, imageHeight); - final imageWidget = CustomPaint( - size: imageSize, - painter: ImagePainter(image: m.image, x: 0, y: 0, scale: s), + print("width: $imageWidth/$imageHeight"); + // final imageWidget = CustomPaint( + // size: imageSize, + // painter: ImagePainter(image: m.image, x: 0, y: 0, scale: s), + // ); + final imageWidget = SizedBox( + width: imageHeight, + height: imageHeight, + child: Obx(() => Texture(textureId: widget.textureId.value)), ); return NotificationListener( @@ -493,9 +517,15 @@ class _ImagePaintState extends State { context, _buildListener(imageWidget), c.size, imageSize)), )); } else { - final imageWidget = CustomPaint( - size: Size(c.size.width, c.size.height), - painter: ImagePainter(image: m.image, x: c.x / s, y: c.y / s, scale: s), + // final imageWidget = CustomPaint( + // size: Size(c.size.width, c.size.height), + // painter: ImagePainter(image: m.image, x: c.x / s, y: c.y / s, scale: s), + // ); + final imageWidget = Center( + child: AspectRatio( + aspectRatio: c.size.width / c.size.height, + child: Obx(() => Texture(textureId: widget.textureId.value)), + ), ); return mouseRegion(child: _buildListener(imageWidget)); } diff --git a/flutter/lib/models/model.dart b/flutter/lib/models/model.dart index 1afb5b147..0b6f14636 100644 --- a/flutter/lib/models/model.dart +++ b/flutter/lib/models/model.dart @@ -1446,14 +1446,15 @@ class FFI { } } else if (message is EventToUI_Rgba) { // Fetch the image buffer from rust codes. - final sz = platformFFI.getRgbaSize(id); - if (sz == null || sz == 0) { - return; - } - final rgba = platformFFI.getRgba(id, sz); - if (rgba != null) { - imageModel.onRgba(rgba); - } + // final sz = platformFFI.getRgbaSize(id); + // if (sz == null || sz == 0) { + // return; + // } + // final rgba = platformFFI.getRgba(id, sz); + // if (rgba != null) { + // imageModel.onRgba(rgba); + // } + // imageModel.onRgba(rgba); } } debugPrint('Exit session event loop'); diff --git a/flutter/lib/models/native_model.dart b/flutter/lib/models/native_model.dart index ba62b775e..13f5b4587 100644 --- a/flutter/lib/models/native_model.dart +++ b/flutter/lib/models/native_model.dart @@ -30,6 +30,9 @@ typedef F4Dart = int Function(Pointer); typedef F5 = Void Function(Pointer); typedef F5Dart = void Function(Pointer); typedef HandleEvent = Future Function(Map evt); +// pub fn session_register_texture(id: *const char, ptr: usize) +typedef F6 = Void Function(Pointer, Uint64); +typedef F6Dart = void Function(Pointer, int); /// FFI wrapper around the native Rust core. /// Hides the platform differences. @@ -52,6 +55,8 @@ class PlatformFFI { F3? _session_get_rgba; F4Dart? _session_get_rgba_size; F5Dart? _session_next_rgba; + F6Dart? _session_register_texture; + static get localeName => Platform.localeName; @@ -130,6 +135,13 @@ class PlatformFFI { malloc.free(a); } + void registerTexture(String id, int ptr) { + if (_session_register_texture == null) return; + final a = id.toNativeUtf8(); + _session_register_texture!(a, ptr); + malloc.free(a); + } + /// Init the FFI class, loads the native Rust core library. Future init(String appType) async { _appType = appType; @@ -150,6 +162,7 @@ class PlatformFFI { dylib.lookupFunction("session_get_rgba_size"); _session_next_rgba = dylib.lookupFunction("session_next_rgba"); + _session_register_texture = dylib.lookupFunction("session_register_texture"); try { // SYSTEM user failed _dir = (await getApplicationDocumentsDirectory()).path; diff --git a/src/flutter.rs b/src/flutter.rs index 2888ffe75..f5d764e66 100644 --- a/src/flutter.rs +++ b/src/flutter.rs @@ -8,7 +8,7 @@ use hbb_common::{ bail, config::LocalConfig, get_version_number, message_proto::*, rendezvous_proto::ConnType, ResultType, }; -use libc::c_void; +use libc::{c_void}; use libloading::{Library, Symbol}; use serde_json::json; use std::sync::atomic::{AtomicBool, Ordering}; @@ -37,7 +37,7 @@ lazy_static::lazy_static! { #[cfg(target_os = "macos")] let lib = Library::new("texture_rgba_renderer_plugin.dylib"); #[cfg(target_os = "linux")] - let lib = Library::new("texture_rgba_renderer_plugin.so"); + let lib = Library::new("libtexture_rgba_renderer_plugin.so"); lib.expect("`libtexture_rgba_renderer_plugin` not found, please add `texture_rgba_renderer` in your flutter project") } }; @@ -129,7 +129,8 @@ pub struct FlutterHandler { // We must check the `rgba_valid` before reading [rgba]. pub rgba: Arc>>, pub rgba_valid: Arc, - pub renderer: VideoRenderer, + pub renderer: Arc>, + peer_info: Arc> } pub type FlutterRgbaRendererPluginOnRgba = unsafe extern "C" fn(texture_rgba: *mut c_void, buffer: *const u8, width: c_int, height: c_int); @@ -148,10 +149,12 @@ impl Default for VideoRenderer { fn default() -> Self { unsafe { Self { + ptr: 0, + width: 0, + height: 0, on_rgba_func: TEXURE_RGBA_RENDERER_PLUGIN .get::(b"FlutterRgbaRendererPluginOnRgba") .expect("Symbol FlutterRgbaRendererPluginOnRgba not found."), - ..Default::default() } } } @@ -220,7 +223,7 @@ impl FlutterHandler { } pub fn register_texture(&mut self, ptr: usize) { - self.renderer.ptr = ptr; + self.renderer.write().unwrap().ptr = ptr; } } @@ -381,6 +384,7 @@ impl InvokeUiSession for FlutterHandler { // unused in flutter fn adapt_size(&self) {} + #[inline] fn on_rgba(&self, data: &mut Vec) { // If the current rgba is not fetched by flutter, i.e., is valid. // We give up sending a new event to flutter. @@ -390,13 +394,15 @@ impl InvokeUiSession for FlutterHandler { self.rgba_valid.store(true, Ordering::Relaxed); // Return the rgba buffer to the video handler for reusing allocated rgba buffer. std::mem::swap::>(data, &mut *self.rgba.write().unwrap()); - #[cfg(any(target_os = "android", target_os = "ios"))] if let Some(stream) = &*self.event_stream.read().unwrap() { stream.add(EventToUI::Rgba); } #[cfg(not(any(target_os = "android", target_os = "ios")))] - self.renderer + { + self.renderer.read().unwrap() .on_rgba(self.rgba.read().unwrap().as_ptr()); + self.next_rgba(); + } } fn set_peer_info(&self, pi: &PeerInfo) { @@ -410,6 +416,9 @@ impl InvokeUiSession for FlutterHandler { features.insert("privacy_mode", 0); } let features = serde_json::ser::to_string(&features).unwrap_or("".to_owned()); + *self.peer_info.write().unwrap() = pi.clone(); + let curr_display = &pi.displays[pi.current_display as usize]; + self.renderer.write().unwrap().set_size(curr_display.width, curr_display.height); self.push_event( "peer_info", vec![ @@ -426,6 +435,7 @@ impl InvokeUiSession for FlutterHandler { } fn set_displays(&self, displays: &Vec) { + self.peer_info.write().unwrap().displays = displays.clone(); self.push_event( "sync_peer_info", vec![("displays", &Self::make_displays_msg(displays))], @@ -457,6 +467,8 @@ impl InvokeUiSession for FlutterHandler { } fn switch_display(&self, display: &SwitchDisplay) { + let curr_display = &self.peer_info.read().unwrap().displays[display.display as usize]; + self.renderer.write().unwrap().set_size(curr_display.width, curr_display.height); self.push_event( "switch_display", vec![ @@ -521,7 +533,7 @@ impl InvokeUiSession for FlutterHandler { } #[inline] - fn next_rgba(&mut self) { + fn next_rgba(&self) { self.rgba_valid.store(false, Ordering::Relaxed); } } diff --git a/src/ui/remote.rs b/src/ui/remote.rs index 4794efb65..7b31c84e9 100644 --- a/src/ui/remote.rs +++ b/src/ui/remote.rs @@ -298,7 +298,7 @@ impl InvokeUiSession for SciterHandler { std::ptr::null() } - fn next_rgba(&mut self) {} + fn next_rgba(&self) {} } pub struct SciterSession(Session); diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index 5a83ee572..5fbf2f4e7 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -798,7 +798,7 @@ pub trait InvokeUiSession: Send + Sync + Clone + 'static + Sized + Default { fn on_voice_call_waiting(&self); fn on_voice_call_incoming(&self); fn get_rgba(&self) -> *const u8; - fn next_rgba(&mut self); + fn next_rgba(&self); } impl Deref for Session {