From 7c5a136b6bf72425153366a3cbb1b42120162ade Mon Sep 17 00:00:00 2001 From: csf Date: Thu, 19 May 2022 23:45:44 +0800 Subject: [PATCH] mobile & web rgba stream --- flutter/lib/models/model.dart | 54 +++++++++++++-------------- flutter/lib/models/native_model.dart | 55 +++++++++++++--------------- flutter/lib/models/web_model.dart | 16 ++++---- flutter/web/js/src/globals.js | 13 +------ src/mobile.rs | 21 ++++------- src/mobile_ffi.rs | 47 ++++-------------------- 6 files changed, 76 insertions(+), 130 deletions(-) diff --git a/flutter/lib/models/model.dart b/flutter/lib/models/model.dart index 6d51f57e2..313ab3fc1 100644 --- a/flutter/lib/models/model.dart +++ b/flutter/lib/models/model.dart @@ -17,12 +17,11 @@ import '../widgets/overlay.dart'; import 'native_model.dart' if (dart.library.html) 'web_model.dart'; typedef HandleMsgBox = void Function(Map evt, String id); +bool _waitForImage = false; class FfiModel with ChangeNotifier { PeerInfo _pi = PeerInfo(); Display _display = Display(); - var _decoding = false; - bool _waitForImage = false; var _inputBlocked = false; final _permissions = Map(); bool? _secure; @@ -122,7 +121,6 @@ class FfiModel with ChangeNotifier { void updateEventListener(String peerId) { final void Function(Map) cb = (evt) { - var pos; var name = evt['name']; if (name == 'msgbox') { handleMsgBox(evt, peerId); @@ -138,7 +136,7 @@ class FfiModel with ChangeNotifier { } else if (name == 'cursor_id') { FFI.cursorModel.updateCursorId(evt); } else if (name == 'cursor_position') { - pos = evt; + FFI.cursorModel.updateCursorPosition(evt); } else if (name == 'clipboard') { Clipboard.setData(ClipboardData(text: evt['content'])); } else if (name == 'permission') { @@ -165,31 +163,6 @@ class FfiModel with ChangeNotifier { } else if (name == 'on_client_remove') { FFI.serverModel.onClientRemove(evt); } - if (pos != null) FFI.cursorModel.updateCursorPosition(pos); - if (!_decoding) { - var rgba = PlatformFFI.getRgba(); - if (rgba != null) { - if (_waitForImage) { - _waitForImage = false; - SmartDialog.dismiss(); - } - _decoding = true; - final pid = FFI.id; - ui.decodeImageFromPixels(rgba, _display.width, _display.height, - isWeb ? ui.PixelFormat.rgba8888 : ui.PixelFormat.bgra8888, - (image) { - PlatformFFI.clearRgbaFrame(); - _decoding = false; - if (FFI.id != pid) return; - try { - // my throw exception, because the listener maybe already dispose - FFI.imageModel.update(image); - } catch (e) { - print('update image: $e'); - } - }); - } - } }; PlatformFFI.setEventCallback(cb); } @@ -284,6 +257,29 @@ class ImageModel with ChangeNotifier { ui.Image? get image => _image; + ImageModel() { + PlatformFFI.setRgbaCallback((rgba) { + if (_waitForImage) { + _waitForImage = false; + SmartDialog.dismiss(); + } + final pid = FFI.id; + ui.decodeImageFromPixels( + rgba, + FFI.ffiModel.display.width, + FFI.ffiModel.display.height, + isWeb ? ui.PixelFormat.rgba8888 : ui.PixelFormat.bgra8888, (image) { + if (FFI.id != pid) return; + try { + // my throw exception, because the listener maybe already dispose + FFI.imageModel.update(image); + } catch (e) { + print('update image: $e'); + } + }); + }); + } + void update(ui.Image? image) { if (_image == null && image != null) { if (isDesktop) { diff --git a/flutter/lib/models/native_model.dart b/flutter/lib/models/native_model.dart index 0fe40ddc5..f6824dda8 100644 --- a/flutter/lib/models/native_model.dart +++ b/flutter/lib/models/native_model.dart @@ -19,8 +19,6 @@ class RgbaFrame extends Struct { typedef F2 = Pointer Function(Pointer, Pointer); typedef F3 = void Function(Pointer, Pointer); -typedef F4 = void Function(Pointer); -typedef F5 = Pointer Function(); class PlatformFFI { static Pointer? _lastRgbaFrame; @@ -28,23 +26,8 @@ class PlatformFFI { static String _homeDir = ''; static F2? _getByName; static F3? _setByName; - static F4? _freeRgba; - static F5? _getRgba; static void Function(Map)? _eventCallback; - - static void clearRgbaFrame() { - if (_lastRgbaFrame != null && - _lastRgbaFrame != nullptr && - _freeRgba != null) _freeRgba!(_lastRgbaFrame!); - } - - static Uint8List? getRgba() { - if (_getRgba == null) return null; - _lastRgbaFrame = _getRgba!(); - if (_lastRgbaFrame == null || _lastRgbaFrame == nullptr) return null; - final ref = _lastRgbaFrame!.ref; - return Uint8List.sublistView(ref.data.asTypedList(ref.len)); - } + static void Function(Uint8List)? _rgbaCallback; static Future getVersion() async { PackageInfo packageInfo = await PackageInfo.fromPlatform(); @@ -85,9 +68,6 @@ class PlatformFFI { _setByName = dylib.lookupFunction, Pointer), F3>( 'set_by_name'); - _freeRgba = dylib - .lookupFunction), F4>('free_rgba'); - _getRgba = dylib.lookupFunction('get_rgba'); _dir = (await getApplicationDocumentsDirectory()).path; _startListenEvent(RustdeskImpl(dylib)); try { @@ -119,23 +99,38 @@ class PlatformFFI { version = await getVersion(); } - static void _startListenEvent(RustdeskImpl rustdeskImpl) async { - await for (final message in rustdeskImpl.startEventStream()) { - if (_eventCallback != null) { - try { - Map event = json.decode(message); - _eventCallback!(event); - } catch (e) { - print('json.decode fail(): $e'); + static void _startListenEvent(RustdeskImpl rustdeskImpl) { + () async { + await for (final message in rustdeskImpl.startEventStream()) { + if (_eventCallback != null) { + try { + Map event = json.decode(message); + _eventCallback!(event); + } catch (e) { + print('json.decode fail(): $e'); + } } } - } + }(); + () async { + await for (final rgba in rustdeskImpl.startRgbaStream()) { + if (_rgbaCallback != null) { + _rgbaCallback!(rgba); + } else { + rgba.clear(); + } + } + }(); } static void setEventCallback(void Function(Map) fun) async { _eventCallback = fun; } + static void setRgbaCallback(void Function(Uint8List) fun) async { + _rgbaCallback = fun; + } + static void startDesktopWebListener() {} static void stopDesktopWebListener() {} diff --git a/flutter/lib/models/web_model.dart b/flutter/lib/models/web_model.dart index fd7a017bf..d9668272a 100644 --- a/flutter/lib/models/web_model.dart +++ b/flutter/lib/models/web_model.dart @@ -10,12 +10,6 @@ final List> mouseListeners = []; final List> keyListeners = []; class PlatformFFI { - static void clearRgbaFrame() {} - - static Uint8List? getRgba() { - return context.callMethod('getRgba'); - } - static String getByName(String name, [String arg = '']) { return context.callMethod('getByName', [name, arg]); } @@ -31,7 +25,7 @@ class PlatformFFI { version = getByName('version'); } - static void setEventCallback(void Function(Map) fun) async { + static void setEventCallback(void Function(Map) fun) { context["onGlobalEvent"] = (String message) { try { Map event = json.decode(message); @@ -42,6 +36,14 @@ class PlatformFFI { }; } + static void setRgbaCallback(void Function(Uint8List) fun) { + context["onRgba"] = (Uint8List? rgba) { + if (rgba != null) { + fun(rgba); + } + }; + } + static void startDesktopWebListener() { mouseListeners.add( window.document.onContextMenu.listen((evt) => evt.preventDefault())); diff --git a/flutter/web/js/src/globals.js b/flutter/web/js/src/globals.js index e43bff342..953add18d 100644 --- a/flutter/web/js/src/globals.js +++ b/flutter/web/js/src/globals.js @@ -6,14 +6,7 @@ import { checkIfRetry, version } from "./gen_js_from_hbb"; import { initZstd, translate } from "./common"; import PCMPlayer from "pcm-player"; -var currentFrame = undefined; - window.curConn = undefined; -window.getRgba = () => { - const tmp = currentFrame; - currentFrame = undefined; - return tmp || null; -} window.isMobile = () => { return /(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|ipad|iris|kindle|Android|Silk|lge |maemo|midp|mmp|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows (ce|phone)|xda|xiino/i.test(navigator.userAgent) || /1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s\-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|\-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw\-(n|u)|c55\/|capi|ccwa|cdm\-|cell|chtm|cldc|cmd\-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc\-s|devi|dica|dmob|do(c|p)o|ds(12|\-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(\-|_)|g1 u|g560|gene|gf\-5|g\-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd\-(m|p|t)|hei\-|hi(pt|ta)|hp( i|ip)|hs\-c|ht(c(\-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i\-(20|go|ma)|i230|iac( |\-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc\-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|\-[a-w])|libw|lynx|m1\-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m\-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(\-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)\-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|\-([1-8]|c))|phil|pire|pl(ay|uc)|pn\-2|po(ck|rt|se)|prox|psio|pt\-g|qa\-a|qc(07|12|21|32|60|\-[2-7]|i\-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h\-|oo|p\-)|sdk\/|se(c(\-|0|1)|47|mc|nd|ri)|sgh\-|shar|sie(\-|m)|sk\-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h\-|v\-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl\-|tdg\-|tel(i|m)|tim\-|t\-mo|to(pl|sh)|ts(70|m\-|m3|m5)|tx\-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|\-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(\-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas\-|your|zeto|zte\-/i.test(navigator.userAgent.substr(0, 4)); @@ -81,7 +74,7 @@ export function draw(frame) { for (let i = 0; i < size; i += row) { flipPixels.set(pixels.subarray(i, i + row), end - i); } - currentFrame = flipPixels; + onRgba(flipPixels); testSpeed[1] += new Date().getTime() - tm0; testSpeed[0] += 1; if (testSpeed[0] > 30) { @@ -116,7 +109,6 @@ export function getConn() { } export async function startConn(id) { - currentFrame = undefined; setByName('remote_id', id); await curConn.start(id); } @@ -124,7 +116,6 @@ export async function startConn(id) { export function close() { getConn()?.close(); setConn(undefined); - currentFrame = undefined; } export function newConn() { @@ -339,7 +330,7 @@ export function playAudio(packet) { window.init = async () => { if (yuvWorker) { yuvWorker.onmessage = (e) => { - currentFrame = e.data; + onRgba(e.data); } } opusWorker.onmessage = (e) => { diff --git a/src/mobile.rs b/src/mobile.rs index 38e80a8e3..6487d0d25 100644 --- a/src/mobile.rs +++ b/src/mobile.rs @@ -1,5 +1,5 @@ use crate::client::*; -use flutter_rust_bridge::StreamSink; +use flutter_rust_bridge::{StreamSink, ZeroCopyBuffer}; use hbb_common::{ allow_err, compress::decompress, @@ -24,7 +24,8 @@ use std::{ lazy_static::lazy_static! { static ref SESSION: Arc>> = Default::default(); - pub static ref EVENT_STREAM: RwLock>> = Default::default(); // rust to dart channel + pub static ref EVENT_STREAM: RwLock>> = Default::default(); // rust to dart event channel + pub static ref RGBA_STREAM: RwLock>>>> = Default::default(); // rust to dart rgba (big u8 list) channel } #[derive(Clone, Default)] @@ -33,7 +34,6 @@ pub struct Session { sender: Arc>>>, lc: Arc>, events2ui: Arc>>, - rgba: Arc>>>, } impl Session { @@ -89,14 +89,6 @@ impl Session { } } - pub fn rgba() -> Option> { - if let Some(session) = SESSION.read().unwrap().as_ref() { - session.rgba.write().unwrap().take() - } else { - None - } - } - pub fn pop_event() -> Option { if let Some(session) = SESSION.read().unwrap().as_ref() { session.events2ui.write().unwrap().pop_front() @@ -607,8 +599,11 @@ impl Connection { if !self.first_frame { self.first_frame = true; } - if let Ok(true) = self.video_handler.handle_frame(vf) { - *self.session.rgba.write().unwrap() = Some(self.video_handler.rgb.clone()); + if let (Ok(true), Some(s)) = ( + self.video_handler.handle_frame(vf), + RGBA_STREAM.read().unwrap().as_ref(), + ) { + s.add(ZeroCopyBuffer(self.video_handler.rgb.clone())); } } Some(message::Union::hash(hash)) => { diff --git a/src/mobile_ffi.rs b/src/mobile_ffi.rs index 8fadf1cce..2d1b90e7c 100644 --- a/src/mobile_ffi.rs +++ b/src/mobile_ffi.rs @@ -1,7 +1,7 @@ use crate::client::file_trait::FileManager; use crate::mobile::connection_manager::{self, get_clients_length, get_clients_state}; use crate::mobile::{self, make_fd_to_json, Session}; -use flutter_rust_bridge::StreamSink; +use flutter_rust_bridge::{StreamSink, ZeroCopyBuffer}; use hbb_common::ResultType; use hbb_common::{ config::{self, Config, LocalConfig, PeerConfig, ONLINE}, @@ -40,6 +40,11 @@ pub fn start_event_stream(s: StreamSink) -> ResultType<()> { Ok(()) } +pub fn start_rgba_stream(s: StreamSink>>) -> ResultType<()> { + let _ = mobile::RGBA_STREAM.write().unwrap().insert(s); + Ok(()) +} + #[no_mangle] unsafe extern "C" fn get_by_name(name: *const c_char, arg: *const c_char) -> *const c_char { let mut res = "".to_owned(); @@ -359,7 +364,7 @@ unsafe extern "C" fn set_by_name(name: *const c_char, value: *const c_char) { m.get("file_num"), m.get("need_override"), m.get("remember"), - m.get("is_upload") + m.get("is_upload"), ) { Session::set_confirm_override_file( id.parse().unwrap_or(0), @@ -507,44 +512,6 @@ unsafe extern "C" fn set_by_name(name: *const c_char, value: *const c_char) { } } -#[repr(C)] -struct RgbaFrame { - len: u32, - data: *mut u8, -} - -#[no_mangle] -unsafe extern "C" fn get_rgba() -> *mut RgbaFrame { - if let Some(mut vec) = Session::rgba() { - if vec.is_empty() { - return std::ptr::null_mut(); - } - assert!(vec.len() == vec.capacity()); - vec.shrink_to_fit(); - let data = vec.as_mut_ptr(); - let len = vec.len(); - std::mem::forget(vec); - Box::into_raw(Box::new(RgbaFrame { - len: len as _, - data, - })) - } else { - std::ptr::null_mut() - } -} - -#[no_mangle] -extern "C" fn free_rgba(f: *mut RgbaFrame) { - if f.is_null() { - return; - } - unsafe { - let len = (*f).len as usize; - drop(Vec::from_raw_parts((*f).data, len, len)); - Box::from_raw(f); - } -} - #[cfg(target_os = "android")] pub mod server_side { use hbb_common::{config::Config, log};