mirror/rustdesk
1
0
Fork 0
mirror of https://github.com/rustdesk/rustdesk.git synced 2025-01-19 08:23:01 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat: legacy mode android keyboard support

Browse Source
This commit is contained in:
mcfans 2023-10-19 00:16:22 +08:00
parent bbc241748b
commit 22165ec1a5
8 changed files with 261 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
flutter/android/app/build.gradle
View File

@ -1,3 +1,8 @@
import com.google.protobuf.gradle.*
plugins {
id "com.google.protobuf" version "0.9.4"
}
def keystoreProperties = new Properties()
def keystorePropertiesFile = rootProject.file('key.properties')
if (keystorePropertiesFile.exists()) {
@ -31,10 +36,33 @@ apply plugin: 'com.android.application'
apply plugin: 'kotlin-android'
apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle"
dependencies {
implementation 'com.google.protobuf:protobuf-javalite:3.20.1'
}
protobuf {
protoc {
artifact = 'com.google.protobuf:protoc:3.20.1'
}
generateProtoTasks {
all().configureEach { task ->
task.builtins {
java {
option "lite"
}
}
}
}
}
android {
compileSdkVersion 33
sourceSets {
main.java.srcDirs += 'src/main/kotlin'
main.proto.srcDirs += '../../../libs/hbb_common/protos'
main.proto.includes += "message.proto"
}
compileOptions {

92
flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt
View File

@ -11,13 +11,18 @@ import android.accessibilityservice.GestureDescription
import android.graphics.Path
import android.os.Build
import android.os.Bundle
import android.os.Handler
import android.os.Looper
import android.util.Log
import android.widget.EditText
import android.view.accessibility.AccessibilityEvent
import android.view.accessibility.AccessibilityNodeInfo
import androidx.annotation.RequiresApi
import java.util.*
import kotlin.math.abs
import kotlin.math.max
import hbb.MessageOuterClass.KeyEvent;
import hbb.KeyEventConverter;
const val LIFT_DOWN = 9
const val LIFT_MOVE = 8
@ -60,6 +65,8 @@ class InputService : AccessibilityService() {
private var isWheelActionsPolling = false
private var isWaitingLongPress = false
private var fakeEditTextForTextStateCalculation: EditText? = null
@RequiresApi(Build.VERSION_CODES.N)
fun onMouseInput(mask: Int, _x: Int, _y: Int) {
val x = max(0, _x)
@ -255,20 +262,87 @@ class InputService : AccessibilityService() {
}
@RequiresApi(Build.VERSION_CODES.N)
fun onTextInput(str: String) {
findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.let {
val arguments = Bundle()
arguments.putCharSequence(
AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE,
str
)
it.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments)
fun onKeyEvent(data: ByteArray) {
val keyEvent = KeyEvent.parseFrom(data);
val handler = Handler(Looper.getMainLooper())
handler.post {
findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.let { node ->
val text = node.getText()
val isShowingHint = node.isShowingHintText()
var textSelectionStart = node.getTextSelectionStart()
var textSelectionEnd = node.getTextSelectionEnd()
if (text != null) {
if (textSelectionStart > text.length) {
textSelectionStart = text.length
}
if (textSelectionEnd > text.length) {
textSelectionEnd = text.length
}
if (textSelectionStart > textSelectionEnd) {
textSelectionStart = textSelectionEnd
}
}
if (keyEvent.hasSeq()) {
val seq = keyEvent.getSeq()
var newText = ""
if ((textSelectionStart == -1) || (textSelectionEnd == -1)) {
newText = seq
} else {
newText = text.let {
it.substring(0, textSelectionStart) + seq + it.substring(textSelectionStart)
}
}
val arguments = Bundle()
arguments.putCharSequence(
AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE,
newText
)
node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments)
} else {
KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event ->
Log.d(logTag, "event $event text $text start $textSelectionStar end $textSelectionEnd")
if (isShowingHint) {
this.fakeEditTextForTextStateCalculation?.setText(null)
} else {
this.fakeEditTextForTextStateCalculation?.setText(text)
}
if (textSelectionStart != -1 && textSelectionEnd != -1) {
this.fakeEditTextForTextStateCalculation?.setSelection(
textSelectionStart,
textSelectionEnd
)
}
this.fakeEditTextForTextStateCalculation?.dispatchKeyEvent(event)
this.fakeEditTextForTextStateCalculation?.getText()?.let { newText ->
val arguments = Bundle()
arguments.putCharSequence(
AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE,
newText.toString()
)
node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments)
}
}
}
}
}
}
override fun onAccessibilityEvent(event: AccessibilityEvent) {
}
override fun onServiceConnected() {
super.onServiceConnected()
ctx = this
fakeEditTextForTextStateCalculation = EditText(this)
Log.d(logTag, "onServiceConnected!")
}
@ -277,7 +351,5 @@ class InputService : AccessibilityService() {
super.onDestroy()
}
override fun onAccessibilityEvent(event: AccessibilityEvent?) {}
override fun onInterrupt() {}
}

115
flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt Normal file
View File

@ -0,0 +1,115 @@
package hbb;
import android.view.KeyEvent
import android.view.KeyCharacterMap
import hbb.MessageOuterClass.ControlKey
object KeyEventConverter {
fun toAndroidKeyEvent(keyEventProto: hbb.MessageOuterClass.KeyEvent): KeyEvent {
var chrValue = 0
var modifiers = 0
android.util.Log.d(tag, "proto: $keyEventProto")
if (keyEventProto.hasUnicode()) {
chrValue =
}
if (keyEventProto.hasChr()) {
chrValue = convertUnicodeToKeyCode(keyEventProto.getChr() as Int)
} else if (keyEventProto.hasControlKey()) {
chrValue = convertControlKeyToKeyCode(keyEventProto.getControlKey())
}
var modifiersList = keyEventProto.getModifiersList()
if (modifiersList != null) {
for (modifier in keyEventProto.getModifiersList()) {
val modifierValue = convertModifier(modifier)
modifiers = modifiers and modifierValue
}
}
var action = 0
if (keyEventProto.getDown()) {
action = KeyEvent.ACTION_DOWN
} else {
action = KeyEvent.ACTION_UP
}
return KeyEvent(0, 0, action, chrValue, 0, modifiers)
}
private fun convertModifier(controlKey: hbb.MessageOuterClass.ControlKey): Int {
// Add logic to map ControlKey values to Android KeyEvent key codes.
// You'll need to provide the mapping for each key.
return when (controlKey) {
ControlKey.Alt -> KeyEvent.META_ALT_ON
ControlKey.Control -> KeyEvent.META_CTRL_ON
ControlKey.CapsLock -> KeyEvent.META_CAPS_LOCK_ON
ControlKey.Meta -> KeyEvent.META_META_ON
ControlKey.NumLock -> KeyEvent.META_NUM_LOCK_ON
ControlKey.RShift -> KeyEvent.META_SHIFT_RIGHT_ON
ControlKey.Shift -> KeyEvent.META_SHIFT_ON
ControlKey.RAlt -> KeyEvent.META_ALT_RIGHT_ON
ControlKey.RControl -> KeyEvent.META_CTRL_RIGHT_ON
else -> 0 // Default to unknown.
}
}
private val tag = "KeyEventConverter"
private fun convertUnicodeToKeyCode(unicode: Int): Int {
val charMap = KeyCharacterMap.load(KeyCharacterMap.VIRTUAL_KEYBOARD)
android.util.Log.d(tag, "unicode: $unicode")
val events = charMap.getEvents(charArrayOf(unicode.toChar()))
if (events != null && events.size > 0) {
android.util.Log.d(tag, "keycode ${events[0].keyCode}")
return events[0].keyCode
}
return 0
}
private fun convertControlKeyToKeyCode(controlKey: hbb.MessageOuterClass.ControlKey): Int {
// Add logic to map ControlKey values to Android KeyEvent key codes.
// You'll need to provide the mapping for each key.
return when (controlKey) {
ControlKey.Alt -> KeyEvent.KEYCODE_ALT_LEFT
ControlKey.Backspace -> KeyEvent.KEYCODE_DEL
ControlKey.Control -> KeyEvent.KEYCODE_CTRL_LEFT
ControlKey.CapsLock -> KeyEvent.KEYCODE_CAPS_LOCK
ControlKey.Meta -> KeyEvent.KEYCODE_META_LEFT
ControlKey.NumLock -> KeyEvent.KEYCODE_NUM_LOCK
ControlKey.RShift -> KeyEvent.KEYCODE_SHIFT_RIGHT
ControlKey.Shift -> KeyEvent.KEYCODE_SHIFT_LEFT
ControlKey.RAlt -> KeyEvent.KEYCODE_ALT_RIGHT
ControlKey.RControl -> KeyEvent.KEYCODE_CTRL_RIGHT
ControlKey.DownArrow -> KeyEvent.KEYCODE_DPAD_DOWN
ControlKey.LeftArrow -> KeyEvent.KEYCODE_DPAD_LEFT
ControlKey.RightArrow -> KeyEvent.KEYCODE_DPAD_RIGHT
ControlKey.UpArrow -> KeyEvent.KEYCODE_DPAD_UP
ControlKey.End -> KeyEvent.KEYCODE_MOVE_END
ControlKey.Home -> KeyEvent.KEYCODE_MOVE_HOME
ControlKey.Insert -> KeyEvent.KEYCODE_INSERT
ControlKey.Escape -> KeyEvent.KEYCODE_ESCAPE
ControlKey.F1 -> KeyEvent.KEYCODE_F1
ControlKey.F2 -> KeyEvent.KEYCODE_F2
ControlKey.F3 -> KeyEvent.KEYCODE_F3
ControlKey.F4 -> KeyEvent.KEYCODE_F4
ControlKey.F5 -> KeyEvent.KEYCODE_F5
ControlKey.F6 -> KeyEvent.KEYCODE_F6
ControlKey.F7 -> KeyEvent.KEYCODE_F7
ControlKey.F8 -> KeyEvent.KEYCODE_F8
ControlKey.F9 -> KeyEvent.KEYCODE_F9
ControlKey.F10 -> KeyEvent.KEYCODE_F10
ControlKey.F11 -> KeyEvent.KEYCODE_F11
ControlKey.F12 -> KeyEvent.KEYCODE_F12
ControlKey.Space -> KeyEvent.KEYCODE_SPACE
ControlKey.Tab -> KeyEvent.KEYCODE_TAB
ControlKey.Return -> KeyEvent.KEYCODE_ENTER
ControlKey.Delete -> KeyEvent.KEYCODE_FORWARD_DEL
ControlKey.Clear -> KeyEvent.KEYCODE_CLEAR
ControlKey.Pause -> KeyEvent.KEYCODE_BREAK
else -> 0 // Default to unknown.
}
}
}

5
flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt
View File

@ -44,7 +44,6 @@ import java.nio.ByteBuffer
import kotlin.math.max
import kotlin.math.min
const val DEFAULT_NOTIFY_TITLE = "RustDesk"
const val DEFAULT_NOTIFY_TEXT = "Service is running"
const val DEFAULT_NOTIFY_ID = 1
@ -96,8 +95,8 @@ class MainService : Service() {
@Keep
@RequiresApi(Build.VERSION_CODES.N)
fun rustInputString(input: String) {
InputService.ctx?.onTextInput(input)
fun rustKeyEventInput(input: ByteArray) {
InputService.ctx?.onKeyEvent(input)
}
@Keep

1
flutter/android/app/src/main/res/xml/accessibility_service_config.xml
View File

@ -1,5 +1,6 @@
<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
android:accessibilityEventTypes="typeWindowsChanged"
android:canRetrieveWindowContent="true"
android:accessibilityFlags="flagDefault"
android:notificationTimeout="50"
android:description="@string/accessibility_service_description"

11
libs/scrap/src/android/ffi.rs
View File

@ -173,18 +173,19 @@ pub fn call_main_service_pointer_input(kind: &str, mask: i32, x: i32, y: i32) ->
}
}
pub fn call_main_service_input_string(str: &str) -> JniResult<()> {
pub fn call_main_service_key_event(data: &[u8]) -> JniResult<()> {
if let (Some(jvm), Some(ctx)) = (
JVM.read().unwrap().as_ref(),
MAIN_SERVICE_CTX.read().unwrap().as_ref(),
) {
let mut env = jvm.attach_current_thread_as_daemon()?;
let input_string = env.new_string(str)?;
let data = env.byte_array_from_slice(data)?;
env.call_method(
ctx,
"rustInputString",
"(Ljava/lang/String;)V",
&[JValue::Object(&JObject::from(input_string))],
"rustKeyEventInput",
"([B)V",
&[JValue::Object(&JObject::from(data))],
)?;
return Ok(());
} else {

6
src/keyboard.rs
View File

@ -877,7 +877,7 @@ pub fn map_keyboard_mode(_peer: &str, event: &Event, mut key_event: KeyEvent) ->
Some(key_event)
}
#[cfg(not(any(target_os = "android", target_os = "ios")))]
#[cfg(not(any(target_os = "ios")))]
fn try_fill_unicode(_peer: &str, event: &Event, key_event: &KeyEvent, events: &mut Vec<KeyEvent>) {
match &event.unicode {
Some(unicode_info) => {
@ -1046,11 +1046,11 @@ pub fn translate_keyboard_mode(peer: &str, event: &Event, key_event: KeyEvent) -
events
}
#[cfg(not(any(target_os = "android", target_os = "ios")))]
#[cfg(not(any(target_os = "ios")))]
pub fn keycode_to_rdev_key(keycode: u32) -> Key {
#[cfg(target_os = "windows")]
return rdev::win_key_from_scancode(keycode);
#[cfg(target_os = "linux")]
#[cfg(any(target_os = "linux", target_os = "android"))]
return rdev::linux_key_from_code(keycode);
#[cfg(target_os = "macos")]
return rdev::macos_key_from_code(keycode.try_into().unwrap_or_default());

31
src/server/connection.rs
View File

@ -41,7 +41,7 @@ use hbb_common::{
tokio_util::codec::{BytesCodec, Framed},
};
#[cfg(any(target_os = "android", target_os = "ios"))]
use scrap::android::{call_main_service_pointer_input, call_main_service_input_string};
use scrap::android::{call_main_service_pointer_input, call_main_service_key_event};
use serde_json::{json, value::Value};
use sha2::{Digest, Sha256};
#[cfg(not(any(target_os = "android", target_os = "ios")))]
@ -1725,12 +1725,29 @@ impl Connection {
#[cfg(any(target_os = "ios"))]
Some(message::Union::KeyEvent(..)) => {}
#[cfg(any(target_os = "android"))]
Some(message::Union::KeyEvent(me)) => {
// We can only use seq of key event, android device doesn't support abritrary key stroke.
let seq = me.seq();
let result = call_main_service_input_string(seq);
if let Err(e) = result {
log::debug!("call_main_service_input_string fail:{}", e);
Some(message::Union::KeyEvent(mut me)) => {
let key = match me.mode.enum_value() {
Ok(KeyboardMode::Map) => {
Some(crate::keyboard::keycode_to_rdev_key(me.chr()))
}
Ok(KeyboardMode::Translate) => {
if let Some(key_event::Union::Chr(code)) = me.union {
Some(crate::keyboard::keycode_to_rdev_key(code & 0x0000FFFF))
} else {
None
}
}
_ => None,
};
let encode_result = me.write_to_bytes();
if let Ok(data) = encode_result {
let result = call_main_service_key_event(&data);
if let Err(e) = result {
log::debug!("call_main_service_key_event fail:{}", e);
}
} else {
log::debug!("encode key event fail:{}", encode_result.err().unwrap());
}
}
#[cfg(not(any(target_os = "android", target_os = "ios")))]