From 996a317b57a907f0f9a2b6ae33d8cc1085ca9233 Mon Sep 17 00:00:00 2001 From: Asura Date: Sun, 30 Oct 2022 05:44:33 -0700 Subject: [PATCH] refactor: keyboard listen and grab && fix shift + scroll --- Cargo.lock | 3 +- .../lib/desktop/pages/desktop_home_page.dart | 1 + flutter/pubspec.lock | 4 +- src/client.rs | 2 +- src/flutter.rs | 1 + src/flutter_ffi.rs | 11 +- src/server/input_service.rs | 8 + src/ui_cm_interface.rs | 1 + src/ui_session_interface.rs | 299 +++++++++++------- 9 files changed, 208 insertions(+), 122 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8e7c37981..26ed9455f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4112,7 +4112,7 @@ dependencies = [ [[package]] name = "rdev" version = "0.5.0-2" -source = "git+https://github.com/asur4s/rdev#ea223720532f32652dab803db43f9ce437f2b019" +source = "git+https://github.com/asur4s/rdev#fdcee04f10ea0ef00d36aa612eabb9605ae9f2fc" dependencies = [ "cocoa", "core-foundation 0.9.3", @@ -4123,6 +4123,7 @@ dependencies = [ "inotify", "lazy_static", "libc", + "mio 0.8.4", "strum 0.24.1", "strum_macros 0.24.3", "widestring 1.0.2", diff --git a/flutter/lib/desktop/pages/desktop_home_page.dart b/flutter/lib/desktop/pages/desktop_home_page.dart index a1c69e5a3..5e9f6b935 100644 --- a/flutter/lib/desktop/pages/desktop_home_page.dart +++ b/flutter/lib/desktop/pages/desktop_home_page.dart @@ -428,6 +428,7 @@ class _DesktopHomePageState extends State @override void initState() { super.initState(); + bind.mainStartGrabKeyboard(); Timer(const Duration(seconds: 5), () async { updateUrl = await bind.mainGetSoftwareUpdateUrl(); if (updateUrl.isNotEmpty) setState(() {}); diff --git a/flutter/pubspec.lock b/flutter/pubspec.lock index e9bd5957c..432b15b15 100644 --- a/flutter/pubspec.lock +++ b/flutter/pubspec.lock @@ -250,8 +250,8 @@ packages: dependency: "direct main" description: path: "." - ref: "541f05f766c3f72984ff40b70dd3c7d061f2ce61" - resolved-ref: "541f05f766c3f72984ff40b70dd3c7d061f2ce61" + ref: bf278fc8a8ff787e46fa3ab97674373bfaa20f23 + resolved-ref: bf278fc8a8ff787e46fa3ab97674373bfaa20f23 url: "https://github.com/Kingtous/rustdesk_desktop_multi_window" source: git version: "0.1.0" diff --git a/src/client.rs b/src/client.rs index 9e2627d55..d00df1c48 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1948,5 +1948,5 @@ fn decode_id_pk(signed: &[u8], key: &sign::PublicKey) -> ResultType<(String, [u8 } pub fn disable_keyboard_listening() { - crate::ui_session_interface::KEYBOARD_HOOKED.store(false, Ordering::SeqCst); + crate::ui_session_interface::KEYBOARD_HOOKED.store(true, Ordering::SeqCst); } diff --git a/src/flutter.rs b/src/flutter.rs index 5b8504b4d..18425476d 100644 --- a/src/flutter.rs +++ b/src/flutter.rs @@ -404,6 +404,7 @@ pub fn session_start_(id: &str, event_stream: StreamSink) -> ResultTy *session.event_stream.write().unwrap() = Some(event_stream); let session = session.clone(); std::thread::spawn(move || { + // if flutter : disable keyboard listen crate::client::disable_keyboard_listening(); io_loop(session); }); diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index 1c53829f1..f20eb6153 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -21,7 +21,8 @@ use crate::{ client::file_trait::FileManager, flutter::{make_fd_to_json, session_add, session_start_}, }; - +#[cfg(not(any(target_os = "android", target_os = "ios")))] +use crate::ui_session_interface::CUR_SESSION; fn initialize(app_dir: &str) { *config::APP_DIR.write().unwrap() = app_dir.to_owned(); #[cfg(target_os = "android")] @@ -224,10 +225,13 @@ pub fn session_handle_flutter_key_event( } pub fn session_enter_or_leave(id: String, enter: bool) { + #[cfg(not(any(target_os = "android", target_os = "ios")))] if let Some(session) = SESSIONS.read().unwrap().get(&id) { if enter { + *CUR_SESSION.lock().unwrap() = Some(session.clone()); session.enter(); } else { + *CUR_SESSION.lock().unwrap() = None; session.leave(); } } @@ -1015,6 +1019,11 @@ pub fn main_is_installed() -> SyncReturn { SyncReturn(is_installed()) } +pub fn main_start_grab_keyboard(){ + #[cfg(not(any(target_os = "android", target_os = "ios")))] + crate::ui_session_interface::global_grab_keyboard(); +} + pub fn main_is_installed_lower_version() -> SyncReturn { SyncReturn(is_installed_lower_version()) } diff --git a/src/server/input_service.rs b/src/server/input_service.rs index 546013673..d91e9a799 100644 --- a/src/server/input_service.rs +++ b/src/server/input_service.rs @@ -429,6 +429,13 @@ fn handle_mouse_(evt: &MouseEvent, conn: i32) { x = -x; y = -y; } + + // fix shift + scroll(down/up) + #[cfg(target_os = "macos")] + if evt.modifiers.contains(&EnumOrUnknown::new(ControlKey::Shift)){ + x = y; + y = 0; + } if x != 0 { en.mouse_scroll_x(x); } @@ -621,6 +628,7 @@ fn rdev_key_click(key: RdevKey) { } fn sync_status(evt: &KeyEvent) -> (bool, bool) { + /* todo! Shift+delete */ let mut en = ENIGO.lock().unwrap(); // remote caps status diff --git a/src/ui_cm_interface.rs b/src/ui_cm_interface.rs index eb1b0e013..61366594c 100644 --- a/src/ui_cm_interface.rs +++ b/src/ui_cm_interface.rs @@ -339,6 +339,7 @@ impl IpcTaskRunner { Data::FS(fs) => { handle_fs(fs, &mut write_jobs, &self.tx).await; } + #[cfg(windows)] Data::ClipbaordFile(_clip) => { #[cfg(windows)] { diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index a085e9822..2f5543ead 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -13,25 +13,25 @@ use async_trait::async_trait; use hbb_common::config::{Config, LocalConfig, PeerConfig}; use hbb_common::rendezvous_proto::ConnType; use hbb_common::tokio::{self, sync::mpsc}; +use hbb_common::{allow_err, message_proto::*}; +use hbb_common::{fs, get_version_number, log, Stream}; #[cfg(not(any(target_os = "android", target_os = "ios")))] use rdev::Keyboard as RdevKeyboard; use rdev::{Event, EventType, EventType::*, Key as RdevKey, KeyboardState}; - -use hbb_common::{allow_err, message_proto::*}; -use hbb_common::{fs, get_version_number, log, Stream}; use std::collections::{HashMap, HashSet}; use std::ops::{Deref, DerefMut}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, RwLock}; +use std::time::Duration; /// IS_IN KEYBOARD_HOOKED sciter only pub static IS_IN: AtomicBool = AtomicBool::new(false); -pub static KEYBOARD_HOOKED: AtomicBool = AtomicBool::new(true); -pub static HOTKEY_HOOK_ENABLED: AtomicBool = AtomicBool::new(false); -#[cfg(target_os = "linux")] -use rdev::IS_GRAB; +pub static KEYBOARD_HOOKED: AtomicBool = AtomicBool::new(false); +pub static HOTKEY_HOOKED: AtomicBool = AtomicBool::new(false); #[cfg(windows)] static mut IS_ALT_GR: bool = false; +#[cfg(feature = "flutter")] +use crate::flutter::FlutterHandler; lazy_static::lazy_static! { static ref TO_RELEASE: Arc>> = Arc::new(Mutex::new(HashSet::::new())); @@ -42,6 +42,12 @@ lazy_static::lazy_static! { static ref KEYBOARD: Arc> = Arc::new(Mutex::new(RdevKeyboard::new().unwrap())); } +#[cfg(feature = "flutter")] +#[cfg(not(any(target_os = "android", target_os = "ios")))] +lazy_static::lazy_static! { + pub static ref CUR_SESSION: Arc>>> = Default::default(); +} + lazy_static::lazy_static! { static ref MUTEX_SPECIAL_KEYS: Mutex> = { let mut m = HashMap::new(); @@ -365,7 +371,6 @@ impl Session { if get_key_state(enigo::Key::NumLock) { key_event.modifiers.push(ControlKey::NumLock.into()); } - self.send_key_event(key_event, KeyboardMode::Map); } @@ -669,6 +674,8 @@ impl Session { } } + #[cfg(not(any(target_os = "android", target_os = "ios")))] + let (alt, ctrl, shift, command) = get_all_hotkey_state(alt, ctrl, shift, command); self.legacy_modifiers(&mut key_event, alt, ctrl, shift, command); if down_or_up == true { @@ -689,12 +696,13 @@ impl Session { #[cfg(not(windows))] let key = self.convert_numpad_keys(key); + let mut to_release = TO_RELEASE.lock().unwrap(); match mode { KeyboardMode::Map => { if down_or_up == true { - TO_RELEASE.lock().unwrap().insert(key); + to_release.insert(key); } else { - TO_RELEASE.lock().unwrap().remove(&key); + to_release.remove(&key); } self.map_keyboard_mode(down_or_up, key, Some(evt)); } @@ -774,30 +782,33 @@ impl Session { } pub fn enter(&self) { - HOTKEY_HOOK_ENABLED.store(true, Ordering::SeqCst); + IS_IN.store(true, Ordering::SeqCst); #[cfg(target_os = "linux")] - unsafe { - IS_GRAB.store(true, Ordering::SeqCst); - } + self.grab_hotkeys(true); #[cfg(windows)] crate::platform::windows::stop_system_key_propagate(true); - IS_IN.store(true, Ordering::SeqCst); } pub fn leave(&self) { - HOTKEY_HOOK_ENABLED.store(false, Ordering::SeqCst); + IS_IN.store(false, Ordering::SeqCst); #[cfg(target_os = "linux")] - unsafe { - IS_GRAB.store(false, Ordering::SeqCst); - } + self.grab_hotkeys(false); for key in TO_RELEASE.lock().unwrap().iter() { self.map_keyboard_mode(false, *key, None) } #[cfg(windows)] crate::platform::windows::stop_system_key_propagate(false); - IS_IN.store(false, Ordering::SeqCst); + } + + #[cfg(target_os = "linux")] + pub fn grab_hotkeys(&self, _grab: bool) { + if _grab { + rdev::enable_grab().ok(); + } else { + rdev::disable_grab().ok(); + } } pub fn handle_flutter_key_event( @@ -846,6 +857,9 @@ impl Session { shift: bool, command: bool, ) { + if HOTKEY_HOOKED.load(Ordering::SeqCst) { + return; + } let chars: Vec = name.chars().collect(); if chars.len() == 1 { let key = Key::_Raw(chars[0] as _); @@ -1205,11 +1219,23 @@ impl Interface for Session { crate::platform::windows::add_recent_document(&path); } } - - #[cfg(not(any(target_os = "android", target_os = "ios")))] - self.start_keyboard_hook(); - #[cfg(not(any(target_os = "android", target_os = "ios")))] - self.start_hotkey_grab(); + // only run in sciter + #[cfg(not(feature = "flutter"))] + { + // rdev::grab and rdev::listen use the same api in macOS & Windows + /* todo! Unused */ + #[cfg(not(any( + target_os = "android", + target_os = "ios", + target_os = "macos", + target_os = "windows", + target_os = "linux", + )))] + self.start_keyboard_hook(); + /* todo! (sciter) Only one device can be connected at the same time in linux */ + #[cfg(not(any(target_os = "android", target_os = "ios")))] + self.start_grab_hotkey(); + } } async fn handle_hash(&mut self, pass: &str, hash: Hash, peer: &mut Stream) { @@ -1252,106 +1278,81 @@ impl Interface for Session { #[cfg(not(any(target_os = "android", target_os = "ios")))] impl Session { - fn handle_hot_key_event(&self, event: Event) { - // keyboard long press - match event.event_type { - EventType::KeyPress(k) => { - if MUTEX_SPECIAL_KEYS.lock().unwrap().contains_key(&k) { - if *MUTEX_SPECIAL_KEYS.lock().unwrap().get(&k).unwrap() { - return; - } - MUTEX_SPECIAL_KEYS.lock().unwrap().insert(k, true); - } - } - EventType::KeyRelease(k) => { - if MUTEX_SPECIAL_KEYS.lock().unwrap().contains_key(&k) { - MUTEX_SPECIAL_KEYS.lock().unwrap().insert(k, false); - } - } + fn handle_hotkey_event(&self, event: Event) { + // if is long press, don't do anything. + if is_long_press(&event) { + return; + } + + let (key, down) = match event.event_type { + EventType::KeyPress(key) => (key, true), + EventType::KeyRelease(key) => (key, false), _ => return, }; - // keyboard short press - match event.event_type { - EventType::KeyPress(key) => { - self.key_down_or_up(true, key, event); - } - EventType::KeyRelease(key) => { - self.key_down_or_up(false, key, event); - } - _ => {} - } + self.key_down_or_up(down, key, event); } - fn start_hotkey_grab(&self) { + #[allow(dead_code)] + fn start_grab_hotkey(&self) { + if self.is_port_forward() || self.is_file_transfer() { + return; + } #[cfg(target_os = "linux")] if !*IS_X11.lock().unwrap() { return; } - if self.is_port_forward() || self.is_file_transfer() { + if HOTKEY_HOOKED.swap(true, Ordering::SeqCst) { return; } + + log::info!("starting grab hotkeys"); let me = self.clone(); - log::info!("hotkey grabing"); - std::thread::spawn(move || { - std::env::set_var("KEYBOARD_ONLY", "y"); - - let func = move |event: Event| { - #[cfg(any(target_os = "windows", target_os = "macos"))] - if !HOTKEY_HOOK_ENABLED.load(Ordering::SeqCst) { - return Some(event); - }; - match event.event_type { - EventType::KeyPress(_key) | EventType::KeyRelease(_key) => { - #[cfg(any(target_os = "windows", target_os = "macos"))] - if MUTEX_SPECIAL_KEYS.lock().unwrap().contains_key(&_key) { - me.handle_hot_key_event(event); - return None; - } else { - return Some(event); - } - - #[cfg(target_os = "linux")] - me.handle_hot_key_event(event); - - None - } - _ => Some(event), + #[cfg(target_os = "linux")] + { + let func = move |event: Event| match event.event_type { + EventType::KeyPress(_key) | EventType::KeyRelease(_key) => { + me.handle_hotkey_event(event); + None } + _ => Some(event), + }; + rdev::start_grab_listen(func) + } + #[cfg(any(target_os = "windows", target_os = "macos"))] + std::thread::spawn(move || { + let func = move |event: Event| match event.event_type { + EventType::KeyPress(key) | EventType::KeyRelease(key) => { + // grab all keys + if !IS_IN.load(Ordering::SeqCst) + || !SERVER_KEYBOARD_ENABLED.load(Ordering::SeqCst) + { + return Some(event); + } else { + me.handle_hotkey_event(event); + return None; + } + } + _ => Some(event), }; - - #[cfg(target_os = "linux")] - { - use rdev::GRABED_KEYS; - GRABED_KEYS.lock().unwrap().insert(RdevKey::ShiftLeft); - GRABED_KEYS.lock().unwrap().insert(RdevKey::ShiftRight); - GRABED_KEYS.lock().unwrap().insert(RdevKey::ControlLeft); - GRABED_KEYS.lock().unwrap().insert(RdevKey::ControlRight); - GRABED_KEYS.lock().unwrap().insert(RdevKey::Alt); - GRABED_KEYS.lock().unwrap().insert(RdevKey::AltGr); - GRABED_KEYS.lock().unwrap().insert(RdevKey::MetaLeft); - GRABED_KEYS.lock().unwrap().insert(RdevKey::MetaRight); - } if let Err(error) = rdev::grab(func) { log::error!("Error: {:?}", error) } }); } + #[allow(dead_code)] fn start_keyboard_hook(&self) { + // only run in sciter if self.is_port_forward() || self.is_file_transfer() { return; } - if !KEYBOARD_HOOKED.load(Ordering::SeqCst) { - return; - } - // rdev::grab and rdev::listen use the same api on macOS - #[cfg(target_os = "macos")] - if HOTKEY_HOOK_ENABLED.load(Ordering::SeqCst) { + if KEYBOARD_HOOKED.swap(true, Ordering::SeqCst) { return; } log::info!("keyboard hooked"); + let me = self.clone(); #[cfg(windows)] crate::platform::windows::enable_lowlevel_keyboard(std::ptr::null_mut() as _); @@ -1360,32 +1361,24 @@ impl Session { std::env::set_var("KEYBOARD_ONLY", "y"); let func = move |evt: Event| { + /* todo! IS_IN can't determine if the user is focused on remote page */ if !IS_IN.load(Ordering::SeqCst) || !SERVER_KEYBOARD_ENABLED.load(Ordering::SeqCst) { return; } - let (_key, down) = match evt.event_type { - KeyPress(k) => { - // keyboard long press - if MUTEX_SPECIAL_KEYS.lock().unwrap().contains_key(&k) { - if *MUTEX_SPECIAL_KEYS.lock().unwrap().get(&k).unwrap() { - return; - } - MUTEX_SPECIAL_KEYS.lock().unwrap().insert(k, true); - } - (k, true) - } - KeyRelease(k) => { - // keyboard long press - if MUTEX_SPECIAL_KEYS.lock().unwrap().contains_key(&k) { - MUTEX_SPECIAL_KEYS.lock().unwrap().insert(k, false); - } - (k, false) - } + if is_long_press(&evt) { + return; + } + let (key, down) = match evt.event_type { + EventType::KeyPress(key) => (key, true), + EventType::KeyRelease(key) => (key, false), _ => return, }; - me.key_down_or_up(down, _key, evt); + me.key_down_or_up(down, key, evt); }; + /* todo!: Shift + a -> AA in sciter + * rdev::listen and rdev::grab both send a + */ if let Err(error) = rdev::listen(func) { log::error!("rdev: {:?}", error); } @@ -1545,7 +1538,11 @@ async fn send_note(url: String, id: String, conn_id: i32, note: String) { } fn get_hotkey_state(key: RdevKey) -> bool { - *MUTEX_SPECIAL_KEYS.lock().unwrap().get(&key).unwrap() + if let Some(&state) = MUTEX_SPECIAL_KEYS.lock().unwrap().get(&key) { + return state; + } else { + return false; + } } fn get_all_hotkey_state( @@ -1565,6 +1562,52 @@ fn get_all_hotkey_state( (alt, ctrl, shift, command) } +#[cfg(feature = "flutter")] +pub fn send_key_event_to_session(event: rdev::Event) { + if let Some(handler) = CUR_SESSION.lock().unwrap().as_ref() { + handler.handle_hotkey_event(event); + } +} + +#[cfg(feature = "flutter")] +pub fn global_grab_keyboard() { + if HOTKEY_HOOKED.swap(true, Ordering::SeqCst) { + return; + } + log::info!("starting global grab keyboard"); + + #[cfg(target_os = "linux")] + { + let func = move |event: Event| match event.event_type { + EventType::KeyPress(_key) | EventType::KeyRelease(_key) => { + send_key_event_to_session(event); + None + } + _ => Some(event), + }; + rdev::start_grab_listen(func) + } + + #[cfg(any(target_os = "windows", target_os = "macos"))] + std::thread::spawn(move || { + let func = move |event: Event| match event.event_type { + EventType::KeyPress(key) | EventType::KeyRelease(key) => { + // grab all keys + if !IS_IN.load(Ordering::SeqCst) { + return Some(event); + } else { + send_key_event_to_session(event); + return None; + } + } + _ => Some(event), + }; + if let Err(error) = rdev::grab(func) { + log::error!("Error: {:?}", error) + } + }); +} + pub fn global_get_keyboard_mode() -> String { return std::env::var("KEYBOARD_MODE") .unwrap_or(String::from("map")) @@ -1574,3 +1617,25 @@ pub fn global_get_keyboard_mode() -> String { pub fn global_save_keyboard_mode(value: String) { std::env::set_var("KEYBOARD_MODE", value); } + +fn is_long_press(event: &Event) -> bool { + let mut keys = MUTEX_SPECIAL_KEYS.lock().unwrap(); + match event.event_type { + EventType::KeyPress(k) => { + if let Some(&state) = keys.get(&k) { + if state == true { + return true; + } else { + keys.insert(k, true); + } + } + } + EventType::KeyRelease(k) => { + if keys.contains_key(&k) { + keys.insert(k, false); + } + } + _ => {} + }; + return false; +}