refactor: keyboard listen and grab && fix shift + scroll

Cargo.lock

@@ -4112,7 +4112,7 @@ dependencies = [
 [[package]]
 name = "rdev"
 version = "0.5.0-2"
-source = "git+https://github.com/asur4s/rdev#ea223720532f32652dab803db43f9ce437f2b019"
+source = "git+https://github.com/asur4s/rdev#fdcee04f10ea0ef00d36aa612eabb9605ae9f2fc"
 dependencies = [
  "cocoa",
  "core-foundation 0.9.3",
@@ -4123,6 +4123,7 @@ dependencies = [
  "inotify",
  "lazy_static",
  "libc",
+ "mio 0.8.4",
  "strum 0.24.1",
  "strum_macros 0.24.3",
  "widestring 1.0.2",

flutter/lib/desktop/pages/desktop_home_page.dart

@@ -428,6 +428,7 @@ class _DesktopHomePageState extends State<DesktopHomePage>
   @override
   void initState() {
     super.initState();
+    bind.mainStartGrabKeyboard();
     Timer(const Duration(seconds: 5), () async {
       updateUrl = await bind.mainGetSoftwareUpdateUrl();
       if (updateUrl.isNotEmpty) setState(() {});

flutter/pubspec.lock

@@ -250,8 +250,8 @@ packages:
     dependency: "direct main"
     description:
       path: "."
-      ref: "541f05f766c3f72984ff40b70dd3c7d061f2ce61"
-      resolved-ref: "541f05f766c3f72984ff40b70dd3c7d061f2ce61"
+      ref: bf278fc8a8ff787e46fa3ab97674373bfaa20f23
+      resolved-ref: bf278fc8a8ff787e46fa3ab97674373bfaa20f23
       url: "https://github.com/Kingtous/rustdesk_desktop_multi_window"
     source: git
     version: "0.1.0"

src/client.rs

@@ -1948,5 +1948,5 @@ fn decode_id_pk(signed: &[u8], key: &sign::PublicKey) -> ResultType<(String, [u8
 }
 
 pub fn disable_keyboard_listening() {
-    crate::ui_session_interface::KEYBOARD_HOOKED.store(false, Ordering::SeqCst);
+    crate::ui_session_interface::KEYBOARD_HOOKED.store(true, Ordering::SeqCst);
 }

src/flutter.rs

@@ -404,6 +404,7 @@ pub fn session_start_(id: &str, event_stream: StreamSink<EventToUI>) -> ResultTy
         *session.event_stream.write().unwrap() = Some(event_stream);
         let session = session.clone();
         std::thread::spawn(move || {
+            // if flutter : disable keyboard listen 
             crate::client::disable_keyboard_listening();
             io_loop(session);
         });

src/flutter_ffi.rs

@@ -21,7 +21,8 @@ use crate::{
     client::file_trait::FileManager,
     flutter::{make_fd_to_json, session_add, session_start_},
 };
-
+#[cfg(not(any(target_os = "android", target_os = "ios")))]
+use crate::ui_session_interface::CUR_SESSION;
 fn initialize(app_dir: &str) {
     *config::APP_DIR.write().unwrap() = app_dir.to_owned();
     #[cfg(target_os = "android")]
@@ -224,10 +225,13 @@ pub fn session_handle_flutter_key_event(
 }
 
 pub fn session_enter_or_leave(id: String, enter: bool) {
+    #[cfg(not(any(target_os = "android", target_os = "ios")))]
     if let Some(session) = SESSIONS.read().unwrap().get(&id) {
         if enter {
+            *CUR_SESSION.lock().unwrap() = Some(session.clone());
             session.enter();
         } else {
+            *CUR_SESSION.lock().unwrap() = None;
             session.leave();
         }
     }
@@ -1015,6 +1019,11 @@ pub fn main_is_installed() -> SyncReturn<bool> {
     SyncReturn(is_installed())
 }
 
+pub fn main_start_grab_keyboard(){
+    #[cfg(not(any(target_os = "android", target_os = "ios")))]
+    crate::ui_session_interface::global_grab_keyboard();
+}
+
 pub fn main_is_installed_lower_version() -> SyncReturn<bool> {
     SyncReturn(is_installed_lower_version())
 }

src/server/input_service.rs

@@ -429,6 +429,13 @@ fn handle_mouse_(evt: &MouseEvent, conn: i32) {
                 x = -x;
                 y = -y;
             }
+
+            // fix shift + scroll(down/up)
+            #[cfg(target_os = "macos")]
+            if evt.modifiers.contains(&EnumOrUnknown::new(ControlKey::Shift)){
+                x = y;
+                y = 0;
+            }
             if x != 0 {
                 en.mouse_scroll_x(x);
             }
@@ -621,6 +628,7 @@ fn rdev_key_click(key: RdevKey) {
 }
 
 fn sync_status(evt: &KeyEvent) -> (bool, bool) {
+    /* todo! Shift+delete */
     let mut en = ENIGO.lock().unwrap();
 
     // remote caps status

src/ui_cm_interface.rs

@@ -339,6 +339,7 @@ impl<T: InvokeUiCM> IpcTaskRunner<T> {
                                 Data::FS(fs) => {
                                     handle_fs(fs, &mut write_jobs, &self.tx).await;
                                 }
+                                #[cfg(windows)]
                                 Data::ClipbaordFile(_clip) => {
                                     #[cfg(windows)]
                                     {

src/ui_session_interface.rs

@@ -13,25 +13,25 @@ use async_trait::async_trait;
 use hbb_common::config::{Config, LocalConfig, PeerConfig};
 use hbb_common::rendezvous_proto::ConnType;
 use hbb_common::tokio::{self, sync::mpsc};
+use hbb_common::{allow_err, message_proto::*};
+use hbb_common::{fs, get_version_number, log, Stream};
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
 use rdev::Keyboard as RdevKeyboard;
 use rdev::{Event, EventType, EventType::*, Key as RdevKey, KeyboardState};
-
-use hbb_common::{allow_err, message_proto::*};
-use hbb_common::{fs, get_version_number, log, Stream};
 use std::collections::{HashMap, HashSet};
 use std::ops::{Deref, DerefMut};
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex, RwLock};
+use std::time::Duration;
 
 /// IS_IN KEYBOARD_HOOKED sciter only
 pub static IS_IN: AtomicBool = AtomicBool::new(false);
-pub static KEYBOARD_HOOKED: AtomicBool = AtomicBool::new(true);
-pub static HOTKEY_HOOK_ENABLED: AtomicBool = AtomicBool::new(false);
-#[cfg(target_os = "linux")]
-use rdev::IS_GRAB;
+pub static KEYBOARD_HOOKED: AtomicBool = AtomicBool::new(false);
+pub static HOTKEY_HOOKED: AtomicBool = AtomicBool::new(false);
 #[cfg(windows)]
 static mut IS_ALT_GR: bool = false;
+#[cfg(feature = "flutter")]
+use crate::flutter::FlutterHandler;
 
 lazy_static::lazy_static! {
     static ref TO_RELEASE: Arc<Mutex<HashSet<RdevKey>>> = Arc::new(Mutex::new(HashSet::<RdevKey>::new()));
@@ -42,6 +42,12 @@ lazy_static::lazy_static! {
     static ref KEYBOARD: Arc<Mutex<RdevKeyboard>> = Arc::new(Mutex::new(RdevKeyboard::new().unwrap()));
 }
 
+#[cfg(feature = "flutter")]
+#[cfg(not(any(target_os = "android", target_os = "ios")))]
+lazy_static::lazy_static! {
+    pub static ref CUR_SESSION: Arc<Mutex<Option<Session<FlutterHandler>>>> = Default::default();
+}
+
 lazy_static::lazy_static! {
     static ref MUTEX_SPECIAL_KEYS: Mutex<HashMap<RdevKey, bool>> = {
         let mut m = HashMap::new();
@@ -365,7 +371,6 @@ impl<T: InvokeUiSession> Session<T> {
         if get_key_state(enigo::Key::NumLock) {
             key_event.modifiers.push(ControlKey::NumLock.into());
         }
-
         self.send_key_event(key_event, KeyboardMode::Map);
     }
 
@@ -669,6 +674,8 @@ impl<T: InvokeUiSession> Session<T> {
             }
         }
 
+        #[cfg(not(any(target_os = "android", target_os = "ios")))]
+        let (alt, ctrl, shift, command) = get_all_hotkey_state(alt, ctrl, shift, command);
         self.legacy_modifiers(&mut key_event, alt, ctrl, shift, command);
 
         if down_or_up == true {
@@ -689,12 +696,13 @@ impl<T: InvokeUiSession> Session<T> {
         #[cfg(not(windows))]
         let key = self.convert_numpad_keys(key);
 
+        let mut to_release = TO_RELEASE.lock().unwrap();
         match mode {
             KeyboardMode::Map => {
                 if down_or_up == true {
-                    TO_RELEASE.lock().unwrap().insert(key);
+                    to_release.insert(key);
                 } else {
-                    TO_RELEASE.lock().unwrap().remove(&key);
+                    to_release.remove(&key);
                 }
                 self.map_keyboard_mode(down_or_up, key, Some(evt));
             }
@@ -774,30 +782,33 @@ impl<T: InvokeUiSession> Session<T> {
     }
 
     pub fn enter(&self) {
-        HOTKEY_HOOK_ENABLED.store(true, Ordering::SeqCst);
+        IS_IN.store(true, Ordering::SeqCst);
         #[cfg(target_os = "linux")]
-        unsafe {
-            IS_GRAB.store(true, Ordering::SeqCst);
-        }
+        self.grab_hotkeys(true);
 
         #[cfg(windows)]
         crate::platform::windows::stop_system_key_propagate(true);
-        IS_IN.store(true, Ordering::SeqCst);
     }
 
     pub fn leave(&self) {
-        HOTKEY_HOOK_ENABLED.store(false, Ordering::SeqCst);
+        IS_IN.store(false, Ordering::SeqCst);
         #[cfg(target_os = "linux")]
-        unsafe {
-            IS_GRAB.store(false, Ordering::SeqCst);
-        }
+        self.grab_hotkeys(false);
 
         for key in TO_RELEASE.lock().unwrap().iter() {
             self.map_keyboard_mode(false, *key, None)
         }
         #[cfg(windows)]
         crate::platform::windows::stop_system_key_propagate(false);
-        IS_IN.store(false, Ordering::SeqCst);
+    }
+
+    #[cfg(target_os = "linux")]
+    pub fn grab_hotkeys(&self, _grab: bool) {
+        if _grab {
+            rdev::enable_grab().ok();
+        } else {
+            rdev::disable_grab().ok();
+        }
     }
 
     pub fn handle_flutter_key_event(
@@ -846,6 +857,9 @@ impl<T: InvokeUiSession> Session<T> {
         shift: bool,
         command: bool,
     ) {
+        if HOTKEY_HOOKED.load(Ordering::SeqCst) {
+            return;
+        }
         let chars: Vec<char> = name.chars().collect();
         if chars.len() == 1 {
             let key = Key::_Raw(chars[0] as _);
@@ -1205,11 +1219,23 @@ impl<T: InvokeUiSession> Interface for Session<T> {
                 crate::platform::windows::add_recent_document(&path);
             }
         }
-
-        #[cfg(not(any(target_os = "android", target_os = "ios")))]
-        self.start_keyboard_hook();
-        #[cfg(not(any(target_os = "android", target_os = "ios")))]
-        self.start_hotkey_grab();
+        // only run in sciter
+        #[cfg(not(feature = "flutter"))]
+        {
+            // rdev::grab and rdev::listen use the same api in macOS & Windows
+            /* todo! Unused */
+            #[cfg(not(any(
+                target_os = "android",
+                target_os = "ios",
+                target_os = "macos",
+                target_os = "windows",
+                target_os = "linux",
+            )))]
+            self.start_keyboard_hook();
+            /* todo! (sciter) Only one device can be connected at the same time in linux */
+            #[cfg(not(any(target_os = "android", target_os = "ios")))]
+            self.start_grab_hotkey();
+        }
     }
 
     async fn handle_hash(&mut self, pass: &str, hash: Hash, peer: &mut Stream) {
@@ -1252,106 +1278,81 @@ impl<T: InvokeUiSession> Interface for Session<T> {
 
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
 impl<T: InvokeUiSession> Session<T> {
-    fn handle_hot_key_event(&self, event: Event) {
-        // keyboard long press
-        match event.event_type {
-            EventType::KeyPress(k) => {
-                if MUTEX_SPECIAL_KEYS.lock().unwrap().contains_key(&k) {
-                    if *MUTEX_SPECIAL_KEYS.lock().unwrap().get(&k).unwrap() {
-                        return;
-                    }
-                    MUTEX_SPECIAL_KEYS.lock().unwrap().insert(k, true);
-                }
-            }
-            EventType::KeyRelease(k) => {
-                if MUTEX_SPECIAL_KEYS.lock().unwrap().contains_key(&k) {
-                    MUTEX_SPECIAL_KEYS.lock().unwrap().insert(k, false);
-                }
-            }
+    fn handle_hotkey_event(&self, event: Event) {
+        // if is long press, don't do anything.
+        if is_long_press(&event) {
+            return;
+        }
+
+        let (key, down) = match event.event_type {
+            EventType::KeyPress(key) => (key, true),
+            EventType::KeyRelease(key) => (key, false),
             _ => return,
         };
 
-        // keyboard short press
-        match event.event_type {
-            EventType::KeyPress(key) => {
-                self.key_down_or_up(true, key, event);
-            }
-            EventType::KeyRelease(key) => {
-                self.key_down_or_up(false, key, event);
-            }
-            _ => {}
-        }
+        self.key_down_or_up(down, key, event);
     }
 
-    fn start_hotkey_grab(&self) {
+    #[allow(dead_code)]
+    fn start_grab_hotkey(&self) {
+        if self.is_port_forward() || self.is_file_transfer() {
+            return;
+        }
         #[cfg(target_os = "linux")]
         if !*IS_X11.lock().unwrap() {
             return;
         }
-        if self.is_port_forward() || self.is_file_transfer() {
+        if HOTKEY_HOOKED.swap(true, Ordering::SeqCst) {
             return;
         }
+
+        log::info!("starting grab hotkeys");
         let me = self.clone();
 
-        log::info!("hotkey grabing");
-        std::thread::spawn(move || {
-            std::env::set_var("KEYBOARD_ONLY", "y");
-
-            let func = move |event: Event| {
-                #[cfg(any(target_os = "windows", target_os = "macos"))]
-                if !HOTKEY_HOOK_ENABLED.load(Ordering::SeqCst) {
-                    return Some(event);
-                };
-                match event.event_type {
-                    EventType::KeyPress(_key) | EventType::KeyRelease(_key) => {
-                        #[cfg(any(target_os = "windows", target_os = "macos"))]
-                        if MUTEX_SPECIAL_KEYS.lock().unwrap().contains_key(&_key) {
-                            me.handle_hot_key_event(event);
-                            return None;
-                        } else {
-                            return Some(event);
-                        }
-
-                        #[cfg(target_os = "linux")]
-                        me.handle_hot_key_event(event);
-
-                        None
-                    }
-                    _ => Some(event),
+        #[cfg(target_os = "linux")]
+        {
+            let func = move |event: Event| match event.event_type {
+                EventType::KeyPress(_key) | EventType::KeyRelease(_key) => {
+                    me.handle_hotkey_event(event);
+                    None
                 }
+                _ => Some(event),
+            };
+            rdev::start_grab_listen(func)
+        }
+        #[cfg(any(target_os = "windows", target_os = "macos"))]
+        std::thread::spawn(move || {
+            let func = move |event: Event| match event.event_type {
+                EventType::KeyPress(key) | EventType::KeyRelease(key) => {
+                    // grab all keys
+                    if !IS_IN.load(Ordering::SeqCst)
+                        || !SERVER_KEYBOARD_ENABLED.load(Ordering::SeqCst)
+                    {
+                        return Some(event);
+                    } else {
+                        me.handle_hotkey_event(event);
+                        return None;
+                    }
+                }
+                _ => Some(event),
             };
-
-            #[cfg(target_os = "linux")]
-            {
-                use rdev::GRABED_KEYS;
-                GRABED_KEYS.lock().unwrap().insert(RdevKey::ShiftLeft);
-                GRABED_KEYS.lock().unwrap().insert(RdevKey::ShiftRight);
-                GRABED_KEYS.lock().unwrap().insert(RdevKey::ControlLeft);
-                GRABED_KEYS.lock().unwrap().insert(RdevKey::ControlRight);
-                GRABED_KEYS.lock().unwrap().insert(RdevKey::Alt);
-                GRABED_KEYS.lock().unwrap().insert(RdevKey::AltGr);
-                GRABED_KEYS.lock().unwrap().insert(RdevKey::MetaLeft);
-                GRABED_KEYS.lock().unwrap().insert(RdevKey::MetaRight);
-            }
             if let Err(error) = rdev::grab(func) {
                 log::error!("Error: {:?}", error)
             }
         });
     }
 
+    #[allow(dead_code)]
     fn start_keyboard_hook(&self) {
+        // only run in sciter
         if self.is_port_forward() || self.is_file_transfer() {
             return;
         }
-        if !KEYBOARD_HOOKED.load(Ordering::SeqCst) {
-            return;
-        }
-        // rdev::grab and rdev::listen use the same api on macOS
-        #[cfg(target_os = "macos")]
-        if HOTKEY_HOOK_ENABLED.load(Ordering::SeqCst) {
+        if KEYBOARD_HOOKED.swap(true, Ordering::SeqCst) {
             return;
         }
         log::info!("keyboard hooked");
+
         let me = self.clone();
         #[cfg(windows)]
         crate::platform::windows::enable_lowlevel_keyboard(std::ptr::null_mut() as _);
@@ -1360,32 +1361,24 @@ impl<T: InvokeUiSession> Session<T> {
             std::env::set_var("KEYBOARD_ONLY", "y");
 
             let func = move |evt: Event| {
+                /* todo! IS_IN can't determine if the user is focused on remote page */
                 if !IS_IN.load(Ordering::SeqCst) || !SERVER_KEYBOARD_ENABLED.load(Ordering::SeqCst)
                 {
                     return;
                 }
-                let (_key, down) = match evt.event_type {
-                    KeyPress(k) => {
-                        // keyboard long press
-                        if MUTEX_SPECIAL_KEYS.lock().unwrap().contains_key(&k) {
-                            if *MUTEX_SPECIAL_KEYS.lock().unwrap().get(&k).unwrap() {
-                                return;
-                            }
-                            MUTEX_SPECIAL_KEYS.lock().unwrap().insert(k, true);
-                        }
-                        (k, true)
-                    }
-                    KeyRelease(k) => {
-                        // keyboard long press
-                        if MUTEX_SPECIAL_KEYS.lock().unwrap().contains_key(&k) {
-                            MUTEX_SPECIAL_KEYS.lock().unwrap().insert(k, false);
-                        }
-                        (k, false)
-                    }
+                if is_long_press(&evt) {
+                    return;
+                }
+                let (key, down) = match evt.event_type {
+                    EventType::KeyPress(key) => (key, true),
+                    EventType::KeyRelease(key) => (key, false),
                     _ => return,
                 };
-                me.key_down_or_up(down, _key, evt);
+                me.key_down_or_up(down, key, evt);
             };
+            /* todo!: Shift + a -> AA in sciter
+             * rdev::listen and rdev::grab both send a
+             */
             if let Err(error) = rdev::listen(func) {
                 log::error!("rdev: {:?}", error);
             }
@@ -1545,7 +1538,11 @@ async fn send_note(url: String, id: String, conn_id: i32, note: String) {
 }
 
 fn get_hotkey_state(key: RdevKey) -> bool {
-    *MUTEX_SPECIAL_KEYS.lock().unwrap().get(&key).unwrap()
+    if let Some(&state) = MUTEX_SPECIAL_KEYS.lock().unwrap().get(&key) {
+        return state;
+    } else {
+        return false;
+    }
 }
 
 fn get_all_hotkey_state(
@@ -1565,6 +1562,52 @@ fn get_all_hotkey_state(
     (alt, ctrl, shift, command)
 }
 
+#[cfg(feature = "flutter")]
+pub fn send_key_event_to_session(event: rdev::Event) {
+    if let Some(handler) = CUR_SESSION.lock().unwrap().as_ref() {
+        handler.handle_hotkey_event(event);
+    }
+}
+
+#[cfg(feature = "flutter")]
+pub fn global_grab_keyboard() {
+    if HOTKEY_HOOKED.swap(true, Ordering::SeqCst) {
+        return;
+    }
+    log::info!("starting global grab keyboard");
+
+    #[cfg(target_os = "linux")]
+    {
+        let func = move |event: Event| match event.event_type {
+            EventType::KeyPress(_key) | EventType::KeyRelease(_key) => {
+                send_key_event_to_session(event);
+                None
+            }
+            _ => Some(event),
+        };
+        rdev::start_grab_listen(func)
+    }
+
+    #[cfg(any(target_os = "windows", target_os = "macos"))]
+    std::thread::spawn(move || {
+        let func = move |event: Event| match event.event_type {
+            EventType::KeyPress(key) | EventType::KeyRelease(key) => {
+                // grab all keys
+                if !IS_IN.load(Ordering::SeqCst) {
+                    return Some(event);
+                } else {
+                    send_key_event_to_session(event);
+                    return None;
+                }
+            }
+            _ => Some(event),
+        };
+        if let Err(error) = rdev::grab(func) {
+            log::error!("Error: {:?}", error)
+        }
+    });
+}
+
 pub fn global_get_keyboard_mode() -> String {
     return std::env::var("KEYBOARD_MODE")
         .unwrap_or(String::from("map"))
@@ -1574,3 +1617,25 @@ pub fn global_get_keyboard_mode() -> String {
 pub fn global_save_keyboard_mode(value: String) {
     std::env::set_var("KEYBOARD_MODE", value);
 }
+
+fn is_long_press(event: &Event) -> bool {
+    let mut keys = MUTEX_SPECIAL_KEYS.lock().unwrap();
+    match event.event_type {
+        EventType::KeyPress(k) => {
+            if let Some(&state) = keys.get(&k) {
+                if state == true {
+                    return true;
+                } else {
+                    keys.insert(k, true);
+                }
+            }
+        }
+        EventType::KeyRelease(k) => {
+            if keys.contains_key(&k) {
+                keys.insert(k, false);
+            }
+        }
+        _ => {}
+    };
+    return false;
+}