From 7bbc5183db4c72a33bf466815e31435103af04fd Mon Sep 17 00:00:00 2001 From: dignow Date: Tue, 18 Jul 2023 21:04:12 +0800 Subject: [PATCH 1/5] trackpad scale, mid commit Signed-off-by: dignow --- flutter/lib/models/input_model.dart | 11 ++++++++--- libs/hbb_common/protos/message.proto | 1 + src/client.rs | 13 ++++++++----- src/flutter_ffi.rs | 6 +++++- src/server/input_service.rs | 11 ++++++++++- src/ui_session_interface.rs | 26 ++++++++++++++++++++------ 6 files changed, 52 insertions(+), 16 deletions(-) diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index 1032bbe9d..44a39ffbc 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -55,6 +55,8 @@ class InputModel { final _trackpadSpeed = 0.06; var _trackpadScrollUnsent = Offset.zero; + var _lastScale = 1.0; + // mouse final isPhysicalMouse = false.obs; int _lastButtons = 0; @@ -272,7 +274,7 @@ class InputModel { sendMouse('down', button); } - void tapUp(MouseButtons button) { + void tapUp(MouseButtons button) { sendMouse('up', button); } @@ -337,12 +339,15 @@ class InputModel { } void onPointerPanZoomStart(PointerPanZoomStartEvent e) { + _lastScale = 1.0; _stopFling = true; } // https://docs.flutter.dev/release/breaking-changes/trackpad-gestures - // TODO(support zoom in/out) void onPointerPanZoomUpdate(PointerPanZoomUpdateEvent e) { + final scale = ((e.scale - _lastScale) * 100).toInt(); + _lastScale = e.scale; + final delta = e.panDelta; _trackpadLastDelta = delta; @@ -366,7 +371,7 @@ class InputModel { if (x != 0 || y != 0) { bind.sessionSendMouse( sessionId: sessionId, - msg: '{"type": "trackpad", "x": "$x", "y": "$y"}'); + msg: '{"type": "trackpad", "x": "$x", "y": "$y", "scale": "$scale"}'); } } diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto index e67a5d3b1..20eb08ba5 100644 --- a/libs/hbb_common/protos/message.proto +++ b/libs/hbb_common/protos/message.proto @@ -116,6 +116,7 @@ message MouseEvent { sint32 x = 2; sint32 y = 3; repeated ControlKey modifiers = 4; + sint32 scale = 5; } enum KeyboardMode{ diff --git a/src/client.rs b/src/client.rs index 85047ee29..6937816a4 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1109,7 +1109,8 @@ impl LoginConfigHandler { self.remember = !config.password.is_empty(); self.config = config; let mut sid = rand::random(); - if sid == 0 { // you won the lottery + if sid == 0 { + // you won the lottery sid = 1; } self.session_id = sid; @@ -1928,6 +1929,7 @@ pub fn send_mouse( mask: i32, x: i32, y: i32, + scale: i32, alt: bool, ctrl: bool, shift: bool, @@ -1939,6 +1941,7 @@ pub fn send_mouse( mask, x, y, + scale, ..Default::default() }; if alt { @@ -1971,12 +1974,12 @@ pub fn send_mouse( /// /// * `interface` - The interface for sending data. fn activate_os(interface: &impl Interface) { - send_mouse(0, 0, 0, false, false, false, false, interface); + send_mouse(0, 0, 0, 0, false, false, false, false, interface); std::thread::sleep(Duration::from_millis(50)); - send_mouse(0, 3, 3, false, false, false, false, interface); + send_mouse(0, 3, 3, 0, false, false, false, false, interface); std::thread::sleep(Duration::from_millis(50)); - send_mouse(1 | 1 << 3, 0, 0, false, false, false, false, interface); - send_mouse(2 | 1 << 3, 0, 0, false, false, false, false, interface); + send_mouse(1 | 1 << 3, 0, 0, 0, false, false, false, false, interface); + send_mouse(2 | 1 << 3, 0, 0, 0, false, false, false, false, interface); /* let mut key_event = KeyEvent::new(); // do not use Esc, which has problem with Linux diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index c8de03862..e4564f144 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -1103,8 +1103,12 @@ pub fn session_send_mouse(session_id: SessionID, msg: String) { _ => 0, } << 3; } + let scale = m + .get("scale") + .map(|x| x.parse::().unwrap_or(0)) + .unwrap_or(0); if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { - session.send_mouse(mask, x, y, alt, ctrl, shift, command); + session.send_mouse(mask, x, y, scale, alt, ctrl, shift, command); } } } diff --git a/src/server/input_service.rs b/src/server/input_service.rs index e71d166f5..476aa03b2 100644 --- a/src/server/input_service.rs +++ b/src/server/input_service.rs @@ -759,7 +759,7 @@ pub fn handle_mouse_(evt: &MouseEvent, conn: i32) { let mut en = ENIGO.lock().unwrap(); #[cfg(not(target_os = "macos"))] let mut to_release = Vec::new(); - if evt_type == 1 { + if evt_type == MOUSE_TYPE_DOWN { fix_modifiers(&evt.modifiers[..], &mut en, 0); #[cfg(target_os = "macos")] en.reset_flag(); @@ -883,6 +883,15 @@ pub fn handle_mouse_(evt: &MouseEvent, conn: i32) { for key in to_release { en.key_up(key.clone()); } + handle_mouse_scale(evt.scale); +} + +#[cfg(target_os = "windows")] +fn handle_mouse_scale(scale: i32) { + let mut en = ENIGO.lock().unwrap(); + en.key_down(Key::Control); + en.mouse_scroll_y(scale); + en.key_up(Key::Control); } pub fn is_enter(evt: &KeyEvent) -> bool { diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index ed6052d39..773eee374 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -1,3 +1,4 @@ +use crate::input::{MOUSE_BUTTON_LEFT, MOUSE_TYPE_DOWN, MOUSE_TYPE_UP}; #[cfg(not(any(target_os = "android", target_os = "ios")))] use std::{collections::HashMap, sync::atomic::AtomicBool}; use std::{ @@ -50,7 +51,7 @@ const CHANGE_RESOLUTION_VALID_TIMEOUT_SECS: u64 = 15; #[derive(Clone, Default)] pub struct Session { pub session_id: SessionID, // different from the one in LoginConfigHandler, used for flutter UI message pass - pub id: String, // peer id + pub id: String, // peer id pub password: String, pub args: Vec, pub lc: Arc>, @@ -306,8 +307,7 @@ impl Session { } pub fn get_audit_server(&self, typ: String) -> String { - if LocalConfig::get_option("access_token").is_empty() - { + if LocalConfig::get_option("access_token").is_empty() { return "".to_owned(); } crate::get_audit_server( @@ -695,6 +695,7 @@ impl Session { mask: i32, x: i32, y: i32, + scale: i32, alt: bool, ctrl: bool, shift: bool, @@ -713,15 +714,28 @@ impl Session { let (alt, ctrl, shift, command) = keyboard::client::get_modifiers_state(alt, ctrl, shift, command); - send_mouse(mask, x, y, alt, ctrl, shift, command, self); + send_mouse(mask, x, y, scale, alt, ctrl, shift, command, self); // on macos, ctrl + left button down = right button down, up won't emit, so we need to // emit up myself if peer is not macos // to-do: how about ctrl + left from win to macos if cfg!(target_os = "macos") { let buttons = mask >> 3; let evt_type = mask & 0x7; - if buttons == 1 && evt_type == 1 && ctrl && self.peer_platform() != "Mac OS" { - self.send_mouse((1 << 3 | 2) as _, x, y, alt, ctrl, shift, command); + if buttons == MOUSE_BUTTON_LEFT + && evt_type == MOUSE_TYPE_DOWN + && ctrl + && self.peer_platform() != "Mac OS" + { + self.send_mouse( + (MOUSE_BUTTON_LEFT << 3 | MOUSE_TYPE_UP) as _, + x, + y, + scale, + alt, + ctrl, + shift, + command, + ); } } } From 780cb37d465694a03cbbc6ff0714f1e6cc998d26 Mon Sep 17 00:00:00 2001 From: dignow Date: Tue, 18 Jul 2023 23:10:42 +0800 Subject: [PATCH 2/5] tmp debug Signed-off-by: dignow --- src/server/input_service.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/server/input_service.rs b/src/server/input_service.rs index 476aa03b2..49baeab6b 100644 --- a/src/server/input_service.rs +++ b/src/server/input_service.rs @@ -1,9 +1,9 @@ use super::*; -use crate::input::*; #[cfg(target_os = "macos")] use crate::common::is_server; #[cfg(target_os = "linux")] use crate::common::IS_X11; +use crate::input::*; #[cfg(target_os = "macos")] use dispatch::Queue; use enigo::{Enigo, Key, KeyboardControllable, MouseButton, MouseControllable}; @@ -752,6 +752,14 @@ pub fn handle_mouse_(evt: &MouseEvent, conn: i32) { return; } + if evt.scale != 0 { + #[cfg(target_os = "windows")] + { + handle_scale(evt.scale); + return; + } + } + #[cfg(windows)] crate::platform::windows::try_change_desktop(); let buttons = evt.mask >> 3; @@ -883,14 +891,14 @@ pub fn handle_mouse_(evt: &MouseEvent, conn: i32) { for key in to_release { en.key_up(key.clone()); } - handle_mouse_scale(evt.scale); } #[cfg(target_os = "windows")] -fn handle_mouse_scale(scale: i32) { +fn handle_scale(scale: i32) { let mut en = ENIGO.lock().unwrap(); - en.key_down(Key::Control); - en.mouse_scroll_y(scale); + if en.key_down(Key::Control).is_ok() { + en.mouse_scroll_y(scale); + } en.key_up(Key::Control); } From c86a8fff034c7dd3b3ddf4f33d69eaee5c3538b0 Mon Sep 17 00:00:00 2001 From: dignow Date: Wed, 19 Jul 2023 01:18:10 +0800 Subject: [PATCH 3/5] add pointer device event Signed-off-by: dignow --- flutter/lib/models/input_model.dart | 21 +++++++++- libs/hbb_common/protos/message.proto | 22 ++++++++++- src/client.rs | 46 +++++++++++++++++++--- src/flutter_ffi.rs | 24 +++++++++--- src/ipc.rs | 1 + src/server/connection.rs | 18 +++++++++ src/server/input_service.rs | 57 ++++++++++++++++++++++------ src/server/portable_service.rs | 26 ++++++++++++- src/ui_session_interface.rs | 20 +++++++--- 9 files changed, 204 insertions(+), 31 deletions(-) diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index 44a39ffbc..8fa4d1caf 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -345,9 +345,20 @@ class InputModel { // https://docs.flutter.dev/release/breaking-changes/trackpad-gestures void onPointerPanZoomUpdate(PointerPanZoomUpdateEvent e) { - final scale = ((e.scale - _lastScale) * 100).toInt(); + debugPrint( + 'REMOVE ME =============================== onPointerPanZoomUpdate ${e.scale}'); + final scale = ((e.scale - _lastScale) * 1000).toInt(); _lastScale = e.scale; + if (scale != 0) { + bind.sessionSendPointer( + sessionId: sessionId, + msg: json.encode({ + 'touch': {'scale': scale} + })); + return; + } + final delta = e.panDelta; _trackpadLastDelta = delta; @@ -371,7 +382,7 @@ class InputModel { if (x != 0 || y != 0) { bind.sessionSendMouse( sessionId: sessionId, - msg: '{"type": "trackpad", "x": "$x", "y": "$y", "scale": "$scale"}'); + msg: '{"type": "trackpad", "x": "$x", "y": "$y"}'); } } @@ -427,6 +438,12 @@ class InputModel { } void onPointerPanZoomEnd(PointerPanZoomEndEvent e) { + bind.sessionSendPointer( + sessionId: sessionId, + msg: json.encode({ + 'touch': {'scale': 0} + })); + waitLastFlingDone(); _stopFling = false; diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto index 20eb08ba5..433e186d5 100644 --- a/libs/hbb_common/protos/message.proto +++ b/libs/hbb_common/protos/message.proto @@ -111,12 +111,31 @@ message LoginResponse { } } +message TouchScaleUpdate { + // The delta scale factor relative to the previous scale. + // delta * 1000 + // 0 means scale end + int32 scale = 1; +} + +message TouchEvent { + oneof union { + TouchScaleUpdate scale_update = 1; + } + repeated ControlKey modifiers = 2; +} + +message PointerDeviceEvent { + oneof union { + TouchEvent touch_event = 1; + } +} + message MouseEvent { int32 mask = 1; sint32 x = 2; sint32 y = 3; repeated ControlKey modifiers = 4; - sint32 scale = 5; } enum KeyboardMode{ @@ -683,5 +702,6 @@ message Message { VoiceCallRequest voice_call_request = 23; VoiceCallResponse voice_call_response = 24; PeerInfo peer_info = 25; + PointerDeviceEvent pointer_device_event = 26; } } diff --git a/src/client.rs b/src/client.rs index 6937816a4..798bc83f2 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1929,7 +1929,6 @@ pub fn send_mouse( mask: i32, x: i32, y: i32, - scale: i32, alt: bool, ctrl: bool, shift: bool, @@ -1941,7 +1940,6 @@ pub fn send_mouse( mask, x, y, - scale, ..Default::default() }; if alt { @@ -1968,18 +1966,54 @@ pub fn send_mouse( interface.send(Data::Message(msg_out)); } +#[inline] +pub fn send_touch( + mut evt: TouchEvent, + alt: bool, + ctrl: bool, + shift: bool, + command: bool, + interface: &impl Interface, +) { + let mut msg_out = Message::new(); + if alt { + evt.modifiers.push(ControlKey::Alt.into()); + } + if shift { + evt.modifiers.push(ControlKey::Shift.into()); + } + if ctrl { + evt.modifiers.push(ControlKey::Control.into()); + } + if command { + evt.modifiers.push(ControlKey::Meta.into()); + } + #[cfg(all(target_os = "macos", not(feature = "flutter")))] + if check_scroll_on_mac(mask, x, y) { + let factor = 3; + mouse_event.mask = crate::input::MOUSE_TYPE_TRACKPAD; + mouse_event.x *= factor; + mouse_event.y *= factor; + } + msg_out.set_pointer_device_event(PointerDeviceEvent { + union: Some(pointer_device_event::Union::TouchEvent(evt)), + ..Default::default() + }); + interface.send(Data::Message(msg_out)); +} + /// Activate OS by sending mouse movement. /// /// # Arguments /// /// * `interface` - The interface for sending data. fn activate_os(interface: &impl Interface) { - send_mouse(0, 0, 0, 0, false, false, false, false, interface); + send_mouse(0, 0, 0, false, false, false, false, interface); std::thread::sleep(Duration::from_millis(50)); - send_mouse(0, 3, 3, 0, false, false, false, false, interface); + send_mouse(0, 3, 3, false, false, false, false, interface); std::thread::sleep(Duration::from_millis(50)); - send_mouse(1 | 1 << 3, 0, 0, 0, false, false, false, false, interface); - send_mouse(2 | 1 << 3, 0, 0, 0, false, false, false, false, interface); + send_mouse(1 | 1 << 3, 0, 0, false, false, false, false, interface); + send_mouse(2 | 1 << 3, 0, 0, false, false, false, false, interface); /* let mut key_event = KeyEvent::new(); // do not use Esc, which has problem with Linux diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index e4564f144..011c1b098 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -1069,6 +1069,24 @@ pub fn main_start_dbus_server() { } } +pub fn session_send_pointer(session_id: SessionID, msg: String) { + if let Ok(m) = serde_json::from_str::>(&msg) { + let alt = m.get("alt").is_some(); + let ctrl = m.get("ctrl").is_some(); + let shift = m.get("shift").is_some(); + let command = m.get("command").is_some(); + if let Some(touch_event) = m.get("touch") { + if let Some(scale) = touch_event.get("scale") { + if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { + if let Some(scale) = scale.as_i64() { + session.send_touch_scale(scale as _, alt, ctrl, shift, command); + } + } + } + } + } +} + pub fn session_send_mouse(session_id: SessionID, msg: String) { if let Ok(m) = serde_json::from_str::>(&msg) { let alt = m.get("alt").is_some(); @@ -1103,12 +1121,8 @@ pub fn session_send_mouse(session_id: SessionID, msg: String) { _ => 0, } << 3; } - let scale = m - .get("scale") - .map(|x| x.parse::().unwrap_or(0)) - .unwrap_or(0); if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { - session.send_mouse(mask, x, y, scale, alt, ctrl, shift, command); + session.send_mouse(mask, x, y, alt, ctrl, shift, command); } } } diff --git a/src/ipc.rs b/src/ipc.rs index 526761c8b..9eaade320 100644 --- a/src/ipc.rs +++ b/src/ipc.rs @@ -152,6 +152,7 @@ pub enum DataPortableService { Pong, ConnCount(Option), Mouse((Vec, i32)), + Pointer((Vec, i32)), Key(Vec), RequestStart, WillClose, diff --git a/src/server/connection.rs b/src/server/connection.rs index 4224d670a..c91b813ff 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -115,6 +115,8 @@ enum MessageInput { Mouse((MouseEvent, i32)), #[cfg(not(any(target_os = "android", target_os = "ios")))] Key((KeyEvent, bool)), + #[cfg(not(any(target_os = "android", target_os = "ios")))] + Pointer((PointerDeviceEvent, i32)), BlockOn, BlockOff, #[cfg(all(feature = "flutter", feature = "plugin_framework"))] @@ -668,6 +670,9 @@ impl Connection { handle_key(&msg); } } + MessageInput::Pointer((msg, id)) => { + handle_pointer(&msg, id); + } MessageInput::BlockOn => { if crate::platform::block_input(true) { block_input_mode = true; @@ -1179,6 +1184,12 @@ impl Connection { self.tx_input.send(MessageInput::Mouse((msg, conn_id))).ok(); } + #[inline] + #[cfg(not(any(target_os = "android", target_os = "ios")))] + fn input_pointer(&self, msg: PointerDeviceEvent, conn_id: i32) { + self.tx_input.send(MessageInput::Pointer((msg, conn_id))).ok(); + } + #[inline] #[cfg(not(any(target_os = "android", target_os = "ios")))] fn input_key(&self, msg: KeyEvent, press: bool) { @@ -1577,6 +1588,13 @@ impl Connection { self.input_mouse(me, self.inner.id()); } } + Some(message::Union::PointerDeviceEvent(pde)) => { + #[cfg(not(any(target_os = "android", target_os = "ios")))] + if self.peer_keyboard_enabled() { + MOUSE_MOVE_TIME.store(get_time(), Ordering::SeqCst); + self.input_pointer(pde, self.inner.id()); + } + } #[cfg(any(target_os = "android", target_os = "ios"))] Some(message::Union::KeyEvent(..)) => {} #[cfg(not(any(target_os = "android", target_os = "ios")))] diff --git a/src/server/input_service.rs b/src/server/input_service.rs index 49baeab6b..8562ca3eb 100644 --- a/src/server/input_service.rs +++ b/src/server/input_service.rs @@ -7,7 +7,11 @@ use crate::input::*; #[cfg(target_os = "macos")] use dispatch::Queue; use enigo::{Enigo, Key, KeyboardControllable, MouseButton, MouseControllable}; -use hbb_common::{get_time, protobuf::EnumOrUnknown}; +use hbb_common::{ + get_time, + message_proto::{pointer_device_event::Union::TouchEvent, touch_event::Union::ScaleUpdate}, + protobuf::EnumOrUnknown, +}; use rdev::{self, EventType, Key as RdevKey, KeyCode, RawKey}; #[cfg(target_os = "macos")] use rdev::{CGEventSourceStateID, CGEventTapLocation, VirtualInput}; @@ -523,6 +527,21 @@ pub fn handle_mouse(evt: &MouseEvent, conn: i32) { handle_mouse_(evt, conn); } +// to-do: merge handle_mouse and handle_pointer +pub fn handle_pointer(evt: &PointerDeviceEvent, conn: i32) { + #[cfg(target_os = "macos")] + if !is_server() { + // having GUI, run main GUI thread, otherwise crash + let evt = evt.clone(); + QUEUE.exec_async(move || handle_pointer_(&evt, conn)); + return; + } + #[cfg(windows)] + crate::portable_service::client::handle_pointer(evt, conn); + #[cfg(not(windows))] + handle_pointer_(evt, conn); +} + pub fn fix_key_down_timeout_loop() { std::thread::spawn(move || loop { std::thread::sleep(std::time::Duration::from_millis(10_000)); @@ -743,7 +762,7 @@ fn active_mouse_(conn: i32) -> bool { } } -pub fn handle_mouse_(evt: &MouseEvent, conn: i32) { +pub fn handle_pointer_(evt: &PointerDeviceEvent, conn: i32) { if !active_mouse_(conn) { return; } @@ -752,12 +771,25 @@ pub fn handle_mouse_(evt: &MouseEvent, conn: i32) { return; } - if evt.scale != 0 { - #[cfg(target_os = "windows")] - { - handle_scale(evt.scale); - return; - } + match &evt.union { + Some(TouchEvent(evt)) => match &evt.union { + Some(ScaleUpdate(_scale_evt)) => { + #[cfg(target_os = "windows")] + handle_scale(_scale_evt.scale); + } + _ => {} + }, + _ => {} + } +} + +pub fn handle_mouse_(evt: &MouseEvent, conn: i32) { + if !active_mouse_(conn) { + return; + } + + if EXITING.load(Ordering::SeqCst) { + return; } #[cfg(windows)] @@ -896,10 +928,13 @@ pub fn handle_mouse_(evt: &MouseEvent, conn: i32) { #[cfg(target_os = "windows")] fn handle_scale(scale: i32) { let mut en = ENIGO.lock().unwrap(); - if en.key_down(Key::Control).is_ok() { - en.mouse_scroll_y(scale); + if scale == 0 { + en.key_up(Key::Control); + } else { + if en.key_down(Key::Control).is_ok() { + en.mouse_scroll_y(scale); + } } - en.key_up(Key::Control); } pub fn is_enter(evt: &KeyEvent) -> bool { diff --git a/src/server/portable_service.rs b/src/server/portable_service.rs index b794e9823..c8185b709 100644 --- a/src/server/portable_service.rs +++ b/src/server/portable_service.rs @@ -222,6 +222,8 @@ mod utils { // functions called in separate SYSTEM user process. pub mod server { + use hbb_common::message_proto::PointerDeviceEvent; + use super::*; lazy_static::lazy_static! { @@ -466,6 +468,11 @@ pub mod server { crate::input_service::handle_mouse_(&evt, conn); } } + Pointer((v, conn)) => { + if let Ok(evt) = PointerDeviceEvent::parse_from_bytes(&v) { + crate::input_service::handle_pointer_(&evt, conn); + } + } Key(v) => { if let Ok(evt) = KeyEvent::parse_from_bytes(&v) { crate::input_service::handle_key_(&evt); @@ -499,7 +506,7 @@ pub mod server { // functions called in main process. pub mod client { - use hbb_common::anyhow::Context; + use hbb_common::{anyhow::Context, message_proto::PointerDeviceEvent}; use super::*; @@ -864,6 +871,14 @@ pub mod client { )))) } + fn handle_pointer_(evt: &PointerDeviceEvent, conn: i32) -> ResultType<()> { + let mut v = vec![]; + evt.write_to_vec(&mut v)?; + ipc_send(Data::DataPortableService(DataPortableService::Pointer(( + v, conn, + )))) + } + fn handle_key_(evt: &KeyEvent) -> ResultType<()> { let mut v = vec![]; evt.write_to_vec(&mut v)?; @@ -910,6 +925,15 @@ pub mod client { } } + pub fn handle_pointer(evt: &PointerDeviceEvent, conn: i32) { + if RUNNING.lock().unwrap().clone() { + crate::input_service::update_latest_input_cursor_time(conn); + handle_pointer_(evt, conn).ok(); + } else { + crate::input_service::handle_pointer_(evt, conn); + } + } + pub fn handle_key(evt: &KeyEvent) { if RUNNING.lock().unwrap().clone() { handle_key_(evt).ok(); diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index 773eee374..033033faf 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -35,8 +35,8 @@ use hbb_common::{ use crate::client::io_loop::Remote; use crate::client::{ check_if_retry, handle_hash, handle_login_error, handle_login_from_ui, handle_test_delay, - input_os_password, load_config, send_mouse, start_video_audio_threads, FileManager, Key, - LoginConfigHandler, QualityStatus, KEY_MAP, + input_os_password, load_config, send_mouse, send_touch, start_video_audio_threads, FileManager, + Key, LoginConfigHandler, QualityStatus, KEY_MAP, }; #[cfg(not(any(target_os = "android", target_os = "ios")))] use crate::common::GrabState; @@ -690,12 +690,23 @@ impl Session { self.send_key_event(&key_event); } + pub fn send_touch_scale(&self, scale: i32, alt: bool, ctrl: bool, shift: bool, command: bool) { + let scale_evt = TouchScaleUpdate { + scale, + ..Default::default() + }; + let evt = TouchEvent { + union: Some(touch_event::Union::ScaleUpdate(scale_evt)), + ..Default::default() + }; + send_touch(evt, alt, ctrl, shift, command, self); + } + pub fn send_mouse( &self, mask: i32, x: i32, y: i32, - scale: i32, alt: bool, ctrl: bool, shift: bool, @@ -714,7 +725,7 @@ impl Session { let (alt, ctrl, shift, command) = keyboard::client::get_modifiers_state(alt, ctrl, shift, command); - send_mouse(mask, x, y, scale, alt, ctrl, shift, command, self); + send_mouse(mask, x, y, alt, ctrl, shift, command, self); // on macos, ctrl + left button down = right button down, up won't emit, so we need to // emit up myself if peer is not macos // to-do: how about ctrl + left from win to macos @@ -730,7 +741,6 @@ impl Session { (MOUSE_BUTTON_LEFT << 3 | MOUSE_TYPE_UP) as _, x, y, - scale, alt, ctrl, shift, From b17c6983cec819ff58d25b093100e481269c21fc Mon Sep 17 00:00:00 2001 From: dignow Date: Wed, 19 Jul 2023 01:26:43 +0800 Subject: [PATCH 4/5] win, touch scale, debug Signed-off-by: dignow --- flutter/lib/common/widgets/remote_input.dart | 23 +++++++++++++++----- flutter/lib/models/input_model.dart | 2 -- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/flutter/lib/common/widgets/remote_input.dart b/flutter/lib/common/widgets/remote_input.dart index 8c14bf26c..49c8879f3 100644 --- a/flutter/lib/common/widgets/remote_input.dart +++ b/flutter/lib/common/widgets/remote_input.dart @@ -1,3 +1,5 @@ +import 'dart:convert'; + import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter/gestures.dart'; @@ -230,10 +232,18 @@ class _RawTouchGestureDetectorRegionState return; } if (isDesktop) { - // to-do + final scale = ((d.scale - _scale) * 1000).toInt(); + _scale = d.scale; + + if (scale != 0) { + bind.sessionSendPointer( + sessionId: sessionId, + msg: json.encode({ + 'touch': {'scale': scale} + })); + } } else { // mobile - // to-do: Is this correct? ffi.canvasModel.updateScale(d.scale / _scale); _scale = d.scale; ffi.canvasModel.panX(d.focalPointDelta.dx); @@ -246,14 +256,17 @@ class _RawTouchGestureDetectorRegionState return; } if (isDesktop) { - // to-do + bind.sessionSendPointer( + sessionId: sessionId, + msg: json.encode({ + 'touch': {'scale': 0} + })); } else { // mobile - // to-do: Is this correct? _scale = 1; bind.sessionSetViewStyle(sessionId: sessionId, value: ""); } - inputModel.sendMouse('up', MouseButtons.left); + inputModel.sendMouse('up', MouseButtons.left); } get onHoldDragCancel => null; diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index 8fa4d1caf..6b50aa37f 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -345,8 +345,6 @@ class InputModel { // https://docs.flutter.dev/release/breaking-changes/trackpad-gestures void onPointerPanZoomUpdate(PointerPanZoomUpdateEvent e) { - debugPrint( - 'REMOVE ME =============================== onPointerPanZoomUpdate ${e.scale}'); final scale = ((e.scale - _lastScale) * 1000).toInt(); _lastScale = e.scale; From 31fc018526db48d1ba280ef15367a4cb6f4cbab5 Mon Sep 17 00:00:00 2001 From: dignow Date: Wed, 19 Jul 2023 07:33:35 +0800 Subject: [PATCH 5/5] change message.proto Signed-off-by: dignow --- libs/hbb_common/protos/message.proto | 2 +- src/client.rs | 9 +++------ src/ui_session_interface.rs | 14 +++++++------- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto index 433e186d5..2f732539c 100644 --- a/libs/hbb_common/protos/message.proto +++ b/libs/hbb_common/protos/message.proto @@ -122,13 +122,13 @@ message TouchEvent { oneof union { TouchScaleUpdate scale_update = 1; } - repeated ControlKey modifiers = 2; } message PointerDeviceEvent { oneof union { TouchEvent touch_event = 1; } + repeated ControlKey modifiers = 2; } message MouseEvent { diff --git a/src/client.rs b/src/client.rs index 798bc83f2..ccdae0512 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1967,8 +1967,8 @@ pub fn send_mouse( } #[inline] -pub fn send_touch( - mut evt: TouchEvent, +pub fn send_pointer_device_event( + mut evt: PointerDeviceEvent, alt: bool, ctrl: bool, shift: bool, @@ -1995,10 +1995,7 @@ pub fn send_touch( mouse_event.x *= factor; mouse_event.y *= factor; } - msg_out.set_pointer_device_event(PointerDeviceEvent { - union: Some(pointer_device_event::Union::TouchEvent(evt)), - ..Default::default() - }); + msg_out.set_pointer_device_event(evt); interface.send(Data::Message(msg_out)); } diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index 033033faf..a88a02edb 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -35,8 +35,8 @@ use hbb_common::{ use crate::client::io_loop::Remote; use crate::client::{ check_if_retry, handle_hash, handle_login_error, handle_login_from_ui, handle_test_delay, - input_os_password, load_config, send_mouse, send_touch, start_video_audio_threads, FileManager, - Key, LoginConfigHandler, QualityStatus, KEY_MAP, + input_os_password, load_config, send_mouse, send_pointer_device_event, + start_video_audio_threads, FileManager, Key, LoginConfigHandler, QualityStatus, KEY_MAP, }; #[cfg(not(any(target_os = "android", target_os = "ios")))] use crate::common::GrabState; @@ -695,11 +695,11 @@ impl Session { scale, ..Default::default() }; - let evt = TouchEvent { - union: Some(touch_event::Union::ScaleUpdate(scale_evt)), - ..Default::default() - }; - send_touch(evt, alt, ctrl, shift, command, self); + let mut touch_evt = TouchEvent::new(); + touch_evt.set_scale_update(scale_evt); + let mut evt = PointerDeviceEvent::new(); + evt.set_touch_event(touch_evt); + send_pointer_device_event(evt, alt, ctrl, shift, command, self); } pub fn send_mouse(