feat: mobile map mode (#9717)

Signed-off-by: fufesou <linlong1266@gmail.com>

Cargo.lock

@@ -5205,7 +5205,7 @@ dependencies = [
 [[package]]
 name = "rdev"
 version = "0.5.0-2"
-source = "git+https://github.com/rustdesk-org/rdev#d4c1759926d693ba269e2cb8cf9f87b13e424e4e"
+source = "git+https://github.com/rustdesk-org/rdev#961d25cc00c6b3ef80f444e6a7bed9872e2c35ea"
 dependencies = [
  "cocoa 0.24.1",
  "core-foundation 0.9.4",

flutter/lib/models/input_model.dart

@@ -544,8 +544,7 @@ class InputModel {
       handleKeyDownEventModifiers(e);
     }
 
-    // * Currently mobile does not enable map mode
-    if ((isDesktop || isWebDesktop) && keyboardMode == kKeyMapMode) {
+    if (isMobile || (isDesktop || isWebDesktop) && keyboardMode == kKeyMapMode) {
       // FIXME: e.character is wrong for dead keys, eg: ^ in de
       newKeyboardMode(
           e.character ?? '',

src/flutter_ffi.rs

@@ -222,6 +222,10 @@ pub fn session_get_enable_trusted_devices(session_id: SessionID) -> SyncReturn<b
 
 pub fn session_close(session_id: SessionID) {
     if let Some(session) = sessions::remove_session_by_session_id(&session_id) {
+        // `release_remote_keys` is not required for mobile platforms in common cases.
+        // But we still call it to make the code more stable.
+        #[cfg(any(target_os = "android", target_os = "ios"))]
+        crate::keyboard::release_remote_keys("map");
         session.close_event_stream(session_id);
         session.close();
     }

src/keyboard.rs

@@ -4,6 +4,7 @@ use crate::flutter;
 use crate::platform::windows::{get_char_from_vk, get_unicode_from_vk};
 #[cfg(not(any(feature = "flutter", feature = "cli")))]
 use crate::ui::CUR_SESSION;
+use crate::ui_session_interface::{InvokeUiSession, Session};
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
 use crate::{client::get_key_state, common::GrabState};
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
@@ -105,16 +106,31 @@ pub mod client {
 
     pub fn process_event(keyboard_mode: &str, event: &Event, lock_modes: Option<i32>) {
         let keyboard_mode = get_keyboard_mode_enum(keyboard_mode);
-
         if is_long_press(&event) {
             return;
         }
-
-        for key_event in event_to_key_events(&event, keyboard_mode, lock_modes) {
+        let peer = get_peer_platform().to_lowercase();
+        for key_event in event_to_key_events(peer, &event, keyboard_mode, lock_modes) {
             send_key_event(&key_event);
         }
     }
 
+    pub fn process_event_with_session<T: InvokeUiSession>(
+        keyboard_mode: &str,
+        event: &Event,
+        lock_modes: Option<i32>,
+        session: &Session<T>,
+    ) {
+        let keyboard_mode = get_keyboard_mode_enum(keyboard_mode);
+        if is_long_press(&event) {
+            return;
+        }
+        let peer = session.peer_platform().to_lowercase();
+        for key_event in event_to_key_events(peer, &event, keyboard_mode, lock_modes) {
+            session.send_key_event(&key_event);
+        }
+    }
+
     pub fn get_modifiers_state(
         alt: bool,
         ctrl: bool,
@@ -171,6 +187,7 @@ pub mod client {
         }
     }
 
+    #[cfg(target_os = "android")]
     pub fn map_key_to_control_key(key: &rdev::Key) -> Option<ControlKey> {
         match key {
             Key::Alt => Some(ControlKey::Alt),
@@ -358,7 +375,6 @@ pub fn is_long_press(event: &Event) -> bool {
     return false;
 }
 
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
 pub fn release_remote_keys(keyboard_mode: &str) {
     // todo!: client quit suddenly, how to release keys?
     let to_release = TO_RELEASE.lock().unwrap().clone();
@@ -387,7 +403,6 @@ pub fn get_keyboard_mode_enum(keyboard_mode: &str) -> KeyboardMode {
 }
 
 #[inline]
-#[cfg(not(any(target_os = "ios")))]
 pub fn is_modifier(key: &rdev::Key) -> bool {
     matches!(
         key,
@@ -403,7 +418,6 @@ pub fn is_modifier(key: &rdev::Key) -> bool {
 }
 
 #[inline]
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
 pub fn is_numpad_rdev_key(key: &rdev::Key) -> bool {
     matches!(
         key,
@@ -426,7 +440,6 @@ pub fn is_numpad_rdev_key(key: &rdev::Key) -> bool {
 }
 
 #[inline]
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
 pub fn is_letter_rdev_key(key: &rdev::Key) -> bool {
     matches!(
         key,
@@ -462,7 +475,6 @@ pub fn is_letter_rdev_key(key: &rdev::Key) -> bool {
 // https://github.com/rustdesk/rustdesk/issues/8599
 // We just add these keys as letter keys.
 #[inline]
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
 pub fn is_letter_rdev_key_ex(key: &rdev::Key) -> bool {
     matches!(
         key,
@@ -471,7 +483,6 @@ pub fn is_letter_rdev_key_ex(key: &rdev::Key) -> bool {
 }
 
 #[inline]
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
 fn is_numpad_key(event: &Event) -> bool {
     matches!(event.event_type, EventType::KeyPress(key) | EventType::KeyRelease(key) if is_numpad_rdev_key(&key))
 }
@@ -479,12 +490,10 @@ fn is_numpad_key(event: &Event) -> bool {
 // Check is letter key for lock modes.
 // Only letter keys need to check and send Lock key state.
 #[inline]
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
 fn is_letter_key_4_lock_modes(event: &Event) -> bool {
     matches!(event.event_type, EventType::KeyPress(key) | EventType::KeyRelease(key) if (is_letter_rdev_key(&key) || is_letter_rdev_key_ex(&key)))
 }
 
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
 fn parse_add_lock_modes_modifiers(
     key_event: &mut KeyEvent,
     lock_modes: i32,
@@ -555,10 +564,13 @@ fn update_modifiers_state(event: &Event) {
 }
 
 pub fn event_to_key_events(
+    mut peer: String,
     event: &Event,
     keyboard_mode: KeyboardMode,
     _lock_modes: Option<i32>,
 ) -> Vec<KeyEvent> {
+    peer.retain(|c| !c.is_whitespace());
+
     let mut key_event = KeyEvent::new();
     update_modifiers_state(event);
 
@@ -572,16 +584,9 @@ pub fn event_to_key_events(
         _ => {}
     }
 
-    let mut peer = get_peer_platform().to_lowercase();
-    peer.retain(|c| !c.is_whitespace());
-
     key_event.mode = keyboard_mode.into();
 
-    #[cfg(not(any(target_os = "android", target_os = "ios")))]
-    let mut key_events;
-    #[cfg(any(target_os = "android", target_os = "ios"))]
-    let key_events;
-    key_events = match keyboard_mode {
+    let mut key_events = match keyboard_mode {
         KeyboardMode::Map => map_keyboard_mode(peer.as_str(), event, key_event),
         KeyboardMode::Translate => translate_keyboard_mode(peer.as_str(), event, key_event),
         _ => {
@@ -596,15 +601,14 @@ pub fn event_to_key_events(
         }
     };
 
-    #[cfg(not(any(target_os = "android", target_os = "ios")))]
     let is_numpad_key = is_numpad_key(&event);
-    #[cfg(not(any(target_os = "android", target_os = "ios")))]
     if keyboard_mode != KeyboardMode::Translate || is_numpad_key {
         let is_letter_key = is_letter_key_4_lock_modes(&event);
         for key_event in &mut key_events {
             if let Some(lock_modes) = _lock_modes {
                 parse_add_lock_modes_modifiers(key_event, lock_modes, is_numpad_key, is_letter_key);
             } else {
+                #[cfg(not(any(target_os = "android", target_os = "ios")))]
                 add_lock_modes_modifiers(key_event, is_numpad_key, is_letter_key);
             }
         }
@@ -617,6 +621,7 @@ pub fn send_key_event(key_event: &KeyEvent) {
     if let Some(session) = CUR_SESSION.lock().unwrap().as_ref() {
         session.send_key_event(key_event);
     }
+
     #[cfg(feature = "flutter")]
     if let Some(session) = flutter::get_cur_session() {
         session.send_key_event(key_event);
@@ -936,8 +941,19 @@ fn _map_keyboard_mode(_peer: &str, event: &Event, mut key_event: KeyEvent) -> Op
         _ => event.position_code as _,
     };
     #[cfg(any(target_os = "android", target_os = "ios"))]
-    let keycode = 0;
-
+    let keycode = match _peer {
+        OS_LOWER_WINDOWS => rdev::usb_hid_code_to_win_scancode(event.usb_hid as _)?,
+        OS_LOWER_LINUX => rdev::usb_hid_code_to_linux_code(event.usb_hid as _)?,
+        OS_LOWER_MACOS => {
+            if hbb_common::config::LocalConfig::get_kb_layout_type() == "ISO" {
+                rdev::usb_hid_code_to_macos_iso_code(event.usb_hid as _)?
+            } else {
+                rdev::usb_hid_code_to_macos_code(event.usb_hid as _)?
+            }
+        }
+        OS_LOWER_ANDROID => rdev::usb_hid_code_to_android_key_code(event.usb_hid as _)?,
+        _ => event.usb_hid as _,
+    };
     key_event.set_chr(keycode as _);
     Some(key_event)
 }

src/ui_session_interface.rs

@@ -861,24 +861,13 @@ impl<T: InvokeUiSession> Session<T> {
             platform_code,
             position_code: position_code as _,
             event_type,
+            usb_hid: 0,
             #[cfg(any(target_os = "windows", target_os = "macos"))]
             extra_data: 0,
         };
-        keyboard::client::process_event(keyboard_mode, &event, Some(lock_modes));
+        keyboard::client::process_event_with_session(keyboard_mode, &event, Some(lock_modes), self);
     }
 
-    #[cfg(any(target_os = "ios"))]
-    pub fn handle_flutter_key_event(
-        &self,
-        _keyboard_mode: &str,
-        _character: &str,
-        _usb_hid: i32,
-        _lock_modes: i32,
-        _down_or_up: bool,
-    ) {
-    }
-
-    #[cfg(not(any(target_os = "ios")))]
     pub fn handle_flutter_key_event(
         &self,
         keyboard_mode: &str,
@@ -900,7 +889,6 @@ impl<T: InvokeUiSession> Session<T> {
         }
     }
 
-    #[cfg(not(any(target_os = "ios")))]
     fn _handle_key_flutter_simulation(
         &self,
         _keyboard_mode: &str,
@@ -925,7 +913,6 @@ impl<T: InvokeUiSession> Session<T> {
         self.send_key_event(&key_event);
     }
 
-    #[cfg(not(any(target_os = "ios")))]
     fn _handle_key_non_flutter_simulation(
         &self,
         keyboard_mode: &str,
@@ -936,14 +923,24 @@ impl<T: InvokeUiSession> Session<T> {
     ) {
         let key = rdev::usb_hid_key_from_code(usb_hid as _);
 
+        #[cfg(any(target_os = "android", target_os = "ios"))]
+        let position_code: KeyCode = 0;
+        #[cfg(any(target_os = "android", target_os = "ios"))]
+        let platform_code: KeyCode = 0;
+
         #[cfg(target_os = "windows")]
         let platform_code: u32 = rdev::win_code_from_key(key).unwrap_or(0);
         #[cfg(target_os = "windows")]
         let position_code: KeyCode = rdev::win_scancode_from_key(key).unwrap_or(0) as _;
 
-        #[cfg(not(target_os = "windows"))]
+        #[cfg(not(any(target_os = "windows", target_os = "android", target_os = "ios")))]
         let position_code: KeyCode = rdev::code_from_key(key).unwrap_or(0) as _;
-        #[cfg(not(any(target_os = "windows", target_os = "linux")))]
+        #[cfg(not(any(
+            target_os = "windows",
+            target_os = "android",
+            target_os = "ios",
+            target_os = "linux"
+        )))]
         let platform_code: u32 = position_code as _;
         // For translate mode.
         // We need to set the platform code (keysym) if is AltGr.
@@ -972,10 +969,14 @@ impl<T: InvokeUiSession> Session<T> {
             platform_code,
             position_code: position_code as _,
             event_type,
+            #[cfg(any(target_os = "android", target_os = "ios"))]
+            usb_hid: usb_hid as _,
+            #[cfg(not(any(target_os = "android", target_os = "ios")))]
+            usb_hid: 0,
             #[cfg(any(target_os = "windows", target_os = "macos"))]
             extra_data: 0,
         };
-        keyboard::client::process_event(keyboard_mode, &event, Some(lock_modes));
+        keyboard::client::process_event_with_session(keyboard_mode, &event, Some(lock_modes), self);
     }
 
     // flutter only TODO new input