Merge pull request #5049 from dignow/fix/desktop_touchpad_scale

Fix/desktop touchpad scale

flutter/lib/common/widgets/remote_input.dart

@@ -1,3 +1,5 @@
+import 'dart:convert';
+
 import 'package:flutter/material.dart';
 import 'package:flutter/services.dart';
 import 'package:flutter/gestures.dart';
@@ -230,10 +232,18 @@ class _RawTouchGestureDetectorRegionState
       return;
     }
     if (isDesktop) {
-      // to-do
+      final scale = ((d.scale - _scale) * 1000).toInt();
+      _scale = d.scale;
+
+      if (scale != 0) {
+        bind.sessionSendPointer(
+            sessionId: sessionId,
+            msg: json.encode({
+              'touch': {'scale': scale}
+            }));
+      }
     } else {
       // mobile
-      // to-do: Is this correct?
       ffi.canvasModel.updateScale(d.scale / _scale);
       _scale = d.scale;
       ffi.canvasModel.panX(d.focalPointDelta.dx);
@@ -246,14 +256,17 @@ class _RawTouchGestureDetectorRegionState
       return;
     }
     if (isDesktop) {
-      // to-do
+      bind.sessionSendPointer(
+          sessionId: sessionId,
+          msg: json.encode({
+            'touch': {'scale': 0}
+          }));
     } else {
       // mobile
-      // to-do: Is this correct?
       _scale = 1;
       bind.sessionSetViewStyle(sessionId: sessionId, value: "");
     }
-     inputModel.sendMouse('up', MouseButtons.left);
+    inputModel.sendMouse('up', MouseButtons.left);
   }
 
   get onHoldDragCancel => null;

flutter/lib/models/input_model.dart

@@ -55,6 +55,8 @@ class InputModel {
   final _trackpadSpeed = 0.06;
   var _trackpadScrollUnsent = Offset.zero;
 
+  var _lastScale = 1.0;
+
   // mouse
   final isPhysicalMouse = false.obs;
   int _lastButtons = 0;
@@ -272,7 +274,7 @@ class InputModel {
     sendMouse('down', button);
   }
 
-    void tapUp(MouseButtons button) {
+  void tapUp(MouseButtons button) {
     sendMouse('up', button);
   }
 
@@ -337,12 +339,24 @@ class InputModel {
   }
 
   void onPointerPanZoomStart(PointerPanZoomStartEvent e) {
+    _lastScale = 1.0;
     _stopFling = true;
   }
 
   // https://docs.flutter.dev/release/breaking-changes/trackpad-gestures
-  // TODO(support zoom in/out)
   void onPointerPanZoomUpdate(PointerPanZoomUpdateEvent e) {
+    final scale = ((e.scale - _lastScale) * 1000).toInt();
+    _lastScale = e.scale;
+
+    if (scale != 0) {
+      bind.sessionSendPointer(
+          sessionId: sessionId,
+          msg: json.encode({
+            'touch': {'scale': scale}
+          }));
+      return;
+    }
+
     final delta = e.panDelta;
     _trackpadLastDelta = delta;
 
@@ -422,6 +436,12 @@ class InputModel {
   }
 
   void onPointerPanZoomEnd(PointerPanZoomEndEvent e) {
+    bind.sessionSendPointer(
+        sessionId: sessionId,
+        msg: json.encode({
+          'touch': {'scale': 0}
+        }));
+
     waitLastFlingDone();
     _stopFling = false;
 

libs/hbb_common/protos/message.proto

@@ -111,6 +111,26 @@ message LoginResponse {
   }
 }
 
+message TouchScaleUpdate {
+  // The delta scale factor relative to the previous scale.
+  // delta * 1000
+  // 0 means scale end
+  int32 scale = 1;
+}
+
+message TouchEvent {
+  oneof union {
+    TouchScaleUpdate scale_update = 1;
+  }
+}
+
+message PointerDeviceEvent {
+  oneof union {
+    TouchEvent touch_event = 1;
+  }
+  repeated ControlKey modifiers = 2;
+}
+
 message MouseEvent {
   int32 mask = 1;
   sint32 x = 2;
@@ -682,5 +702,6 @@ message Message {
     VoiceCallRequest voice_call_request = 23;
     VoiceCallResponse voice_call_response = 24;
     PeerInfo peer_info = 25;
+    PointerDeviceEvent pointer_device_event = 26;
   }
 }

src/client.rs

@@ -1109,7 +1109,8 @@ impl LoginConfigHandler {
         self.remember = !config.password.is_empty();
         self.config = config;
         let mut sid = rand::random();
-        if sid == 0 { // you won the lottery
+        if sid == 0 {
+            // you won the lottery
             sid = 1;
         }
         self.session_id = sid;
@@ -1965,6 +1966,39 @@ pub fn send_mouse(
     interface.send(Data::Message(msg_out));
 }
 
+#[inline]
+pub fn send_pointer_device_event(
+    mut evt: PointerDeviceEvent,
+    alt: bool,
+    ctrl: bool,
+    shift: bool,
+    command: bool,
+    interface: &impl Interface,
+) {
+    let mut msg_out = Message::new();
+    if alt {
+        evt.modifiers.push(ControlKey::Alt.into());
+    }
+    if shift {
+        evt.modifiers.push(ControlKey::Shift.into());
+    }
+    if ctrl {
+        evt.modifiers.push(ControlKey::Control.into());
+    }
+    if command {
+        evt.modifiers.push(ControlKey::Meta.into());
+    }
+    #[cfg(all(target_os = "macos", not(feature = "flutter")))]
+    if check_scroll_on_mac(mask, x, y) {
+        let factor = 3;
+        mouse_event.mask = crate::input::MOUSE_TYPE_TRACKPAD;
+        mouse_event.x *= factor;
+        mouse_event.y *= factor;
+    }
+    msg_out.set_pointer_device_event(evt);
+    interface.send(Data::Message(msg_out));
+}
+
 /// Activate OS by sending mouse movement.
 ///
 /// # Arguments

src/flutter_ffi.rs

@@ -1069,6 +1069,24 @@ pub fn main_start_dbus_server() {
     }
 }
 
+pub fn session_send_pointer(session_id: SessionID, msg: String) {
+    if let Ok(m) = serde_json::from_str::<HashMap<String, serde_json::Value>>(&msg) {
+        let alt = m.get("alt").is_some();
+        let ctrl = m.get("ctrl").is_some();
+        let shift = m.get("shift").is_some();
+        let command = m.get("command").is_some();
+        if let Some(touch_event) = m.get("touch") {
+            if let Some(scale) = touch_event.get("scale") {
+                if let Some(session) = SESSIONS.read().unwrap().get(&session_id) {
+                    if let Some(scale) = scale.as_i64() {
+                        session.send_touch_scale(scale as _, alt, ctrl, shift, command);
+                    }
+                }
+            }
+        }
+    }
+}
+
 pub fn session_send_mouse(session_id: SessionID, msg: String) {
     if let Ok(m) = serde_json::from_str::<HashMap<String, String>>(&msg) {
         let alt = m.get("alt").is_some();

src/ipc.rs

@@ -152,6 +152,7 @@ pub enum DataPortableService {
     Pong,
     ConnCount(Option<usize>),
     Mouse((Vec<u8>, i32)),
+    Pointer((Vec<u8>, i32)),
     Key(Vec<u8>),
     RequestStart,
     WillClose,

src/server/connection.rs

@@ -115,6 +115,8 @@ enum MessageInput {
     Mouse((MouseEvent, i32)),
     #[cfg(not(any(target_os = "android", target_os = "ios")))]
     Key((KeyEvent, bool)),
+    #[cfg(not(any(target_os = "android", target_os = "ios")))]
+    Pointer((PointerDeviceEvent, i32)),
     BlockOn,
     BlockOff,
     #[cfg(all(feature = "flutter", feature = "plugin_framework"))]
@@ -668,6 +670,9 @@ impl Connection {
                             handle_key(&msg);
                         }
                     }
+                    MessageInput::Pointer((msg, id)) => {
+                        handle_pointer(&msg, id);
+                    }
                     MessageInput::BlockOn => {
                         if crate::platform::block_input(true) {
                             block_input_mode = true;
@@ -1179,6 +1184,12 @@ impl Connection {
         self.tx_input.send(MessageInput::Mouse((msg, conn_id))).ok();
     }
 
+    #[inline]
+    #[cfg(not(any(target_os = "android", target_os = "ios")))]
+    fn input_pointer(&self, msg: PointerDeviceEvent, conn_id: i32) {
+        self.tx_input.send(MessageInput::Pointer((msg, conn_id))).ok();
+    }
+
     #[inline]
     #[cfg(not(any(target_os = "android", target_os = "ios")))]
     fn input_key(&self, msg: KeyEvent, press: bool) {
@@ -1577,6 +1588,13 @@ impl Connection {
                         self.input_mouse(me, self.inner.id());
                     }
                 }
+                Some(message::Union::PointerDeviceEvent(pde)) => {
+                    #[cfg(not(any(target_os = "android", target_os = "ios")))]
+                    if self.peer_keyboard_enabled() {
+                        MOUSE_MOVE_TIME.store(get_time(), Ordering::SeqCst);
+                        self.input_pointer(pde, self.inner.id());
+                    }
+                }
                 #[cfg(any(target_os = "android", target_os = "ios"))]
                 Some(message::Union::KeyEvent(..)) => {}
                 #[cfg(not(any(target_os = "android", target_os = "ios")))]

src/server/input_service.rs

@@ -1,13 +1,17 @@
 use super::*;
-use crate::input::*;
 #[cfg(target_os = "macos")]
 use crate::common::is_server;
 #[cfg(target_os = "linux")]
 use crate::common::IS_X11;
+use crate::input::*;
 #[cfg(target_os = "macos")]
 use dispatch::Queue;
 use enigo::{Enigo, Key, KeyboardControllable, MouseButton, MouseControllable};
-use hbb_common::{get_time, protobuf::EnumOrUnknown};
+use hbb_common::{
+    get_time,
+    message_proto::{pointer_device_event::Union::TouchEvent, touch_event::Union::ScaleUpdate},
+    protobuf::EnumOrUnknown,
+};
 use rdev::{self, EventType, Key as RdevKey, KeyCode, RawKey};
 #[cfg(target_os = "macos")]
 use rdev::{CGEventSourceStateID, CGEventTapLocation, VirtualInput};
@@ -523,6 +527,21 @@ pub fn handle_mouse(evt: &MouseEvent, conn: i32) {
     handle_mouse_(evt, conn);
 }
 
+// to-do: merge handle_mouse and handle_pointer
+pub fn handle_pointer(evt: &PointerDeviceEvent, conn: i32) {
+    #[cfg(target_os = "macos")]
+    if !is_server() {
+        // having GUI, run main GUI thread, otherwise crash
+        let evt = evt.clone();
+        QUEUE.exec_async(move || handle_pointer_(&evt, conn));
+        return;
+    }
+    #[cfg(windows)]
+    crate::portable_service::client::handle_pointer(evt, conn);
+    #[cfg(not(windows))]
+    handle_pointer_(evt, conn);
+}
+
 pub fn fix_key_down_timeout_loop() {
     std::thread::spawn(move || loop {
         std::thread::sleep(std::time::Duration::from_millis(10_000));
@@ -743,6 +762,27 @@ fn active_mouse_(conn: i32) -> bool {
     }
 }
 
+pub fn handle_pointer_(evt: &PointerDeviceEvent, conn: i32) {
+    if !active_mouse_(conn) {
+        return;
+    }
+
+    if EXITING.load(Ordering::SeqCst) {
+        return;
+    }
+
+    match &evt.union {
+        Some(TouchEvent(evt)) => match &evt.union {
+            Some(ScaleUpdate(_scale_evt)) => {
+                #[cfg(target_os = "windows")]
+                handle_scale(_scale_evt.scale);
+            }
+            _ => {}
+        },
+        _ => {}
+    }
+}
+
 pub fn handle_mouse_(evt: &MouseEvent, conn: i32) {
     if !active_mouse_(conn) {
         return;
@@ -759,7 +799,7 @@ pub fn handle_mouse_(evt: &MouseEvent, conn: i32) {
     let mut en = ENIGO.lock().unwrap();
     #[cfg(not(target_os = "macos"))]
     let mut to_release = Vec::new();
-    if evt_type == 1 {
+    if evt_type == MOUSE_TYPE_DOWN {
         fix_modifiers(&evt.modifiers[..], &mut en, 0);
         #[cfg(target_os = "macos")]
         en.reset_flag();
@@ -885,6 +925,18 @@ pub fn handle_mouse_(evt: &MouseEvent, conn: i32) {
     }
 }
 
+#[cfg(target_os = "windows")]
+fn handle_scale(scale: i32) {
+    let mut en = ENIGO.lock().unwrap();
+    if scale == 0 {
+        en.key_up(Key::Control);
+    } else {
+        if en.key_down(Key::Control).is_ok() {
+            en.mouse_scroll_y(scale);
+        }
+    }
+}
+
 pub fn is_enter(evt: &KeyEvent) -> bool {
     if let Some(key_event::Union::ControlKey(ck)) = evt.union {
         if ck.value() == ControlKey::Return.value() || ck.value() == ControlKey::NumpadEnter.value()

src/server/portable_service.rs

@@ -222,6 +222,8 @@ mod utils {
 
 // functions called in separate SYSTEM user process.
 pub mod server {
+    use hbb_common::message_proto::PointerDeviceEvent;
+
     use super::*;
 
     lazy_static::lazy_static! {
@@ -466,6 +468,11 @@ pub mod server {
                                             crate::input_service::handle_mouse_(&evt, conn);
                                         }
                                     }
+                                    Pointer((v, conn)) => {
+                                        if let Ok(evt) = PointerDeviceEvent::parse_from_bytes(&v) {
+                                            crate::input_service::handle_pointer_(&evt, conn);
+                                        }
+                                    }
                                     Key(v) => {
                                         if let Ok(evt) = KeyEvent::parse_from_bytes(&v) {
                                             crate::input_service::handle_key_(&evt);
@@ -499,7 +506,7 @@ pub mod server {
 
 // functions called in main process.
 pub mod client {
-    use hbb_common::anyhow::Context;
+    use hbb_common::{anyhow::Context, message_proto::PointerDeviceEvent};
 
     use super::*;
 
@@ -864,6 +871,14 @@ pub mod client {
         ))))
     }
 
+    fn handle_pointer_(evt: &PointerDeviceEvent, conn: i32) -> ResultType<()> {
+        let mut v = vec![];
+        evt.write_to_vec(&mut v)?;
+        ipc_send(Data::DataPortableService(DataPortableService::Pointer((
+            v, conn,
+        ))))
+    }
+
     fn handle_key_(evt: &KeyEvent) -> ResultType<()> {
         let mut v = vec![];
         evt.write_to_vec(&mut v)?;
@@ -910,6 +925,15 @@ pub mod client {
         }
     }
 
+    pub fn handle_pointer(evt: &PointerDeviceEvent, conn: i32) {
+        if RUNNING.lock().unwrap().clone() {
+            crate::input_service::update_latest_input_cursor_time(conn);
+            handle_pointer_(evt, conn).ok();
+        } else {
+            crate::input_service::handle_pointer_(evt, conn);
+        }
+    }
+
     pub fn handle_key(evt: &KeyEvent) {
         if RUNNING.lock().unwrap().clone() {
             handle_key_(evt).ok();

src/ui_session_interface.rs

@@ -1,3 +1,4 @@
+use crate::input::{MOUSE_BUTTON_LEFT, MOUSE_TYPE_DOWN, MOUSE_TYPE_UP};
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
 use std::{collections::HashMap, sync::atomic::AtomicBool};
 use std::{
@@ -34,8 +35,8 @@ use hbb_common::{
 use crate::client::io_loop::Remote;
 use crate::client::{
     check_if_retry, handle_hash, handle_login_error, handle_login_from_ui, handle_test_delay,
-    input_os_password, load_config, send_mouse, start_video_audio_threads, FileManager, Key,
-    LoginConfigHandler, QualityStatus, KEY_MAP,
+    input_os_password, load_config, send_mouse, send_pointer_device_event,
+    start_video_audio_threads, FileManager, Key, LoginConfigHandler, QualityStatus, KEY_MAP,
 };
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
 use crate::common::GrabState;
@@ -50,7 +51,7 @@ const CHANGE_RESOLUTION_VALID_TIMEOUT_SECS: u64 = 15;
 #[derive(Clone, Default)]
 pub struct Session<T: InvokeUiSession> {
     pub session_id: SessionID, // different from the one in LoginConfigHandler, used for flutter UI message pass
-    pub id: String, // peer id
+    pub id: String,            // peer id
     pub password: String,
     pub args: Vec<String>,
     pub lc: Arc<RwLock<LoginConfigHandler>>,
@@ -306,8 +307,7 @@ impl<T: InvokeUiSession> Session<T> {
     }
 
     pub fn get_audit_server(&self, typ: String) -> String {
-        if LocalConfig::get_option("access_token").is_empty()
-        {
+        if LocalConfig::get_option("access_token").is_empty() {
             return "".to_owned();
         }
         crate::get_audit_server(
@@ -690,6 +690,18 @@ impl<T: InvokeUiSession> Session<T> {
         self.send_key_event(&key_event);
     }
 
+    pub fn send_touch_scale(&self, scale: i32, alt: bool, ctrl: bool, shift: bool, command: bool) {
+        let scale_evt = TouchScaleUpdate {
+            scale,
+            ..Default::default()
+        };
+        let mut touch_evt = TouchEvent::new();
+        touch_evt.set_scale_update(scale_evt);
+        let mut evt = PointerDeviceEvent::new();
+        evt.set_touch_event(touch_evt);
+        send_pointer_device_event(evt, alt, ctrl, shift, command, self);
+    }
+
     pub fn send_mouse(
         &self,
         mask: i32,
@@ -720,8 +732,20 @@ impl<T: InvokeUiSession> Session<T> {
         if cfg!(target_os = "macos") {
             let buttons = mask >> 3;
             let evt_type = mask & 0x7;
-            if buttons == 1 && evt_type == 1 && ctrl && self.peer_platform() != "Mac OS" {
-                self.send_mouse((1 << 3 | 2) as _, x, y, alt, ctrl, shift, command);
+            if buttons == MOUSE_BUTTON_LEFT
+                && evt_type == MOUSE_TYPE_DOWN
+                && ctrl
+                && self.peer_platform() != "Mac OS"
+            {
+                self.send_mouse(
+                    (MOUSE_BUTTON_LEFT << 3 | MOUSE_TYPE_UP) as _,
+                    x,
+                    y,
+                    alt,
+                    ctrl,
+                    shift,
+                    command,
+                );
             }
         }
     }