update input

android/app/build.gradle

@@ -33,7 +33,7 @@ apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle"
 
 android {
     compileSdkVersion 30
-    // ndkVersion '22.1.7171670' // * 仅个人使用 存在多版本NDK无法自动选择 需要使用此配置指定NDK版本 [CSF]
+     ndkVersion '22.1.7171670' // * 仅个人使用 存在多版本NDK无法自动选择 需要使用此配置指定NDK版本 [CSF]
     sourceSets {
         main.java.srcDirs += 'src/main/kotlin'
     }

android/app/src/main/AndroidManifest.xml

@@ -1,51 +1,76 @@
+<?xml version="1.0" encoding="utf-8"?>
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
     package="com.carriez.flutter_hbb">
-   <application
-        android:label="RustDesk"
-        android:icon="@mipmap/ic_launcher">
-        <activity
-            android:name=".MainActivity"
-            android:launchMode="singleTop"
-            android:theme="@style/LaunchTheme"
-            android:configChanges="orientation|keyboardHidden|keyboard|screenSize|smallestScreenSize|locale|layoutDirection|fontScale|screenLayout|density|uiMode"
-            android:hardwareAccelerated="true"
-            android:windowSoftInputMode="adjustResize">
-            <!-- Specifies an Android theme to apply to this Activity as soon as
-                 the Android process has started. This theme is visible to the user
-                 while the Flutter UI initializes. After that, this theme continues
-                 to determine the Window background behind the Flutter UI. -->
-            <meta-data
-              android:name="io.flutter.embedding.android.NormalTheme"
-              android:resource="@style/NormalTheme"
-              />
-            <!-- Displays an Android View that continues showing the launch screen
-                 Drawable until Flutter paints its first frame, then this splash
-                 screen fades out. A splash screen is useful to avoid any visual
-                 gap between the end of Android's launch screen and the painting of
-                 Flutter's first frame. -->
-            <meta-data
-              android:name="io.flutter.embedding.android.SplashScreenDrawable"
-              android:resource="@drawable/launch_background"
-              />
-            <intent-filter>
-                <action android:name="android.intent.action.MAIN" />
-                <category android:name="android.intent.category.LAUNCHER" />
-            </intent-filter>
-        </activity>
-        <service
-            android:name=".MainService"
-            android:enabled="true"
-            android:foregroundServiceType="mediaProjection"/>
-        <!-- Don't delete the meta-data below.
-             This is used by the Flutter tool to generate GeneratedPluginRegistrant.java -->
-        <meta-data
-            android:name="flutterEmbedding"
-            android:value="2" />
-    </application>
 
     <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
     <uses-permission android:name="android.permission.INTERNET" />
     <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
     <uses-permission android:name="android.permission.RECORD_AUDIO" />
-</manifest>
+
+    <application
+        android:icon="@mipmap/ic_launcher"
+        android:label="RustDesk">
+        <service
+            android:name=".InputService"
+            android:enabled="true"
+            android:exported="false"
+            android:label="RustDesk Input"
+            android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE">
+            <intent-filter>
+                <action android:name="android.accessibilityservice.AccessibilityService" />
+            </intent-filter>
+            <meta-data
+                android:name="android.accessibilityservice"
+                android:resource="@xml/accessibility_service_config" />
+        </service>
+
+        <activity
+            android:name=".MainActivity"
+            android:configChanges="orientation|keyboardHidden|keyboard|screenSize|smallestScreenSize|locale|layoutDirection|fontScale|screenLayout|density|uiMode"
+            android:hardwareAccelerated="true"
+            android:launchMode="singleTop"
+            android:theme="@style/LaunchTheme"
+            android:windowSoftInputMode="adjustResize">
+
+            <!--
+                 Specifies an Android theme to apply to this Activity as soon as
+                 the Android process has started. This theme is visible to the user
+                 while the Flutter UI initializes. After that, this theme continues
+                 to determine the Window background behind the Flutter UI.
+            -->
+            <meta-data
+                android:name="io.flutter.embedding.android.NormalTheme"
+                android:resource="@style/NormalTheme" />
+            <!--
+                 Displays an Android View that continues showing the launch screen
+                 Drawable until Flutter paints its first frame, then this splash
+                 screen fades out. A splash screen is useful to avoid any visual
+                 gap between the end of Android's launch screen and the painting of
+                 Flutter's first frame.
+            -->
+            <meta-data
+                android:name="io.flutter.embedding.android.SplashScreenDrawable"
+                android:resource="@drawable/launch_background" />
+
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+
+        <service
+            android:name=".MainService"
+            android:enabled="true"
+            android:foregroundServiceType="mediaProjection" />
+        <!--
+ Don't delete the meta-data below.
+             This is used by the Flutter tool to generate GeneratedPluginRegistrant.java
+        -->
+        <meta-data
+            android:name="flutterEmbedding"
+            android:value="2" />
+    </application>
+
+</manifest>

android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt

@@ -0,0 +1,106 @@
+package com.carriez.flutter_hbb
+
+import android.accessibilityservice.AccessibilityService
+import android.accessibilityservice.GestureDescription
+import android.content.Context
+import android.graphics.Path
+import android.os.Build
+import android.util.Log
+import android.view.accessibility.AccessibilityEvent
+import androidx.annotation.RequiresApi
+import kotlin.concurrent.thread
+
+class InputService : AccessibilityService() {
+//    companion object {
+//        var inputService:InputService? = null
+//    }
+    private val logTag = "input service"
+    private var leftIsDown = false
+    private var mPath = Path()
+    private var mLastGestureStartTime = 0L
+    private var mouseX = 0
+    private var mouseY = 0
+
+    @RequiresApi(Build.VERSION_CODES.N)
+    fun mouseInput(mask: Int, _x: Int, _y: Int) {
+        Log.w(logTag, "got mouse input:x:$_x ,y:$_y ,mask:$mask ")
+
+        // TODO 临时倍数
+        // TODO 按键抬起按下时候 x y 都是0
+        if ( !(mask == 9 || mask == 10) ) {
+            mouseX = _x * 2
+            mouseY = _y * 2
+        }
+
+        // left button down ,was up
+        if (mask == 9){
+            leftIsDown = true
+            startGesture(mouseX,mouseY)
+        }
+
+        // left down ,was down
+        if (mask == 9){
+            continueGesture(mouseX,mouseY)
+        }
+
+        // left up ,was down
+        if (mask == 10){
+            leftIsDown = false
+            endGesture(mouseX, mouseY)
+        }
+    }
+
+    private fun startGesture(x: Int, y: Int) {
+        mPath = Path()
+        mPath.moveTo(x.toFloat(), y.toFloat())
+        mLastGestureStartTime = System.currentTimeMillis()
+    }
+
+    private fun continueGesture(x: Int, y: Int) {
+        mPath.lineTo(x.toFloat(), y.toFloat())
+    }
+    @RequiresApi(Build.VERSION_CODES.N)
+    private fun endGesture(x: Int, y: Int) {
+        mPath.lineTo(x.toFloat(), y.toFloat())
+        val stroke = GestureDescription.StrokeDescription(
+            mPath,
+            0,
+            System.currentTimeMillis() - mLastGestureStartTime
+        )
+        val builder = GestureDescription.Builder()
+        builder.addStroke(stroke)
+        Log.d(logTag, "end gesture $x $y")
+        dispatchGesture(builder.build(), object : GestureResultCallback() {
+            override fun onCompleted(gestureDescription: GestureDescription) {
+                super.onCompleted(gestureDescription)
+                Log.d(logTag, "滑动成功")
+            }
+
+            override fun onCancelled(gestureDescription: GestureDescription) {
+                super.onCancelled(gestureDescription)
+                Log.d(logTag, "滑动失败 ")
+            }
+        }, null)
+    }
+
+    external fun init(ctx: Context)
+
+    init {
+        System.loadLibrary("rustdesk")
+    }
+
+    private val LOG_TAG = "INPUT_LOG"
+    @RequiresApi(Build.VERSION_CODES.O)
+    override fun onServiceConnected() {
+        super.onServiceConnected()
+        Log.d(LOG_TAG,"onServiceConnected!")
+        init(this)
+    }
+    override fun onAccessibilityEvent(event: AccessibilityEvent?) {
+//        TODO("Not yet implemented")
+    }
+
+    override fun onInterrupt() {
+//        TODO("Not yet implemented")
+    }
+}

android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt

@@ -1,12 +1,14 @@
 package com.carriez.flutter_hbb
 
 import android.app.Activity
+import android.app.AlertDialog
 import android.content.Context
 import android.content.Intent
 import android.media.projection.MediaProjectionManager
 import android.os.Build
 import android.os.Bundle
 import android.os.PersistableBundle
+import android.provider.Settings
 import android.util.Log
 import androidx.annotation.RequiresApi
 import io.flutter.embedding.android.FlutterActivity
@@ -50,6 +52,10 @@ class MainActivity : FlutterActivity() {
                     mStopService()
                     result.success(true)
                 }
+                "checkInput" ->{
+                    checkInput()
+                    result.success(true)
+                }
                 else -> {}
             }
         }
@@ -96,6 +102,25 @@ class MainActivity : FlutterActivity() {
         }
     }
 
+    private fun checkInput() {
+        AlertDialog.Builder(this)
+            .setCancelable(false)
+            .setTitle("检查Input服务")
+            .setMessage("请开启相关服务")
+            .setPositiveButton("Yes") { dialog, which ->
+                val intent = Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS)
+                if (intent.resolveActivity(packageManager) != null) startActivityForResult(
+                    intent,
+                    11
+                ) else AlertDialog.Builder(this)
+                    .setTitle("错误")
+                    .setMessage("无法启动服务")
+                    .show()
+            }
+            .setNegativeButton("No") { dialog, which -> }
+            .show()
+    }
+
     override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {
         super.onActivityResult(requestCode, resultCode, data)
         if (resultCode == Activity.RESULT_OK && data != null) {

android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt

@@ -34,11 +34,11 @@ const val M_KEY_FRAME_RATE = 30
 
 class MainService : Service() {
 
-    fun rustGetRaw():ByteArray{
+    fun rustGetRaw(): ByteArray {
         return rawByteArray!!
     }
 
-    external fun init(ctx:Context)
+    external fun init(ctx: Context)
 
     init {
         System.loadLibrary("rustdesk")
@@ -49,7 +49,7 @@ class MainService : Service() {
     private var surface: Surface? = null
     private val singleThread = Executors.newSingleThreadExecutor()
     private var mEncoder: MediaCodec? = null
-    private var rawByteArray :ByteArray? = null
+    private var rawByteArray: ByteArray? = null
 
     override fun onBind(intent: Intent): IBinder? {
         return null
@@ -90,7 +90,8 @@ class MainService : Service() {
         return super.onStartCommand(intent, flags, startId)
     }
 
-    lateinit var mImageReader:ImageReader // * 注意 这里要成为成员变量，防止被回收 https://www.cnblogs.com/yongdaimi/p/11004560.html
+    lateinit var mImageReader: ImageReader // * 注意 这里要成为成员变量，防止被回收 https://www.cnblogs.com/yongdaimi/p/11004560.html
+
     @SuppressLint("WrongConstant")
     @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
     private fun startRecorder() {
@@ -100,23 +101,23 @@ class MainService : Service() {
             mImageReader =
                 ImageReader.newInstance(FIXED_WIDTH, FIXED_HEIGHT, PixelFormat.RGBA_8888, 2) // 至少是2
             mImageReader.setOnImageAvailableListener({ imageReader: ImageReader ->
-                Log.d(logTag, "on image")
-                    try {
-                        imageReader.acquireLatestImage().use { image ->
-                            if (image == null) return@setOnImageAvailableListener
-                            val planes = image.planes
-                            val buffer = planes[0].buffer
-                            buffer.rewind()
-                            // 这里注意 处理不当会引发OOM
-                            if (rawByteArray == null){
-                                rawByteArray = ByteArray(buffer.capacity())
-                                buffer.get(rawByteArray!!)
-                            }else{
-                                buffer.get(rawByteArray!!)
-                            }
+//                Log.d(logTag, "on image")
+                try {
+                    imageReader.acquireLatestImage().use { image ->
+                        if (image == null) return@setOnImageAvailableListener
+                        val planes = image.planes
+                        val buffer = planes[0].buffer
+                        buffer.rewind()
+                        // 这里注意 处理不当会引发OOM
+                        if (rawByteArray == null) {
+                            rawByteArray = ByteArray(buffer.capacity())
+                            buffer.get(rawByteArray!!)
+                        } else {
+                            buffer.get(rawByteArray!!)
                         }
-                    } catch (ignored: java.lang.Exception) {
                     }
+                } catch (ignored: java.lang.Exception) {
+                }
                 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
                     imageReader.discardFreeBuffers()
                 }

android/app/src/main/res/values/strings.xml

@@ -0,0 +1,4 @@
+<resources>
+    <string name="app_name">RustDesk</string>
+    <string name="accessibility_service_description">测试服务 输入服务</string>
+</resources>

android/app/src/main/res/xml/accessibility_service_config.xml

@@ -0,0 +1,6 @@
+<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
+    android:accessibilityEventTypes="typeWindowsChanged"
+    android:accessibilityFlags="flagDefault"
+    android:notificationTimeout="50"
+    android:description="@string/accessibility_service_description"
+    android:canPerformGestures="true"/>

android/build.gradle

@@ -7,7 +7,7 @@ buildscript {
 
     dependencies {
         classpath 'com.android.tools.build:gradle:4.1.0'
-        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:1.6.10"
         classpath 'com.google.gms:google-services:4.3.3'
     }
 }

lib/home_page.dart

@@ -100,7 +100,8 @@ class _HomePageState extends State<HomePage> {
                 getPeers(),
                 ElevatedButton(onPressed:_toAndroidGetPer, child: Text("获取权限事件")),
                 ElevatedButton(onPressed:_toAndroidStartSer, child: Text("开启录屏服务")),
-                ElevatedButton(onPressed:_toAndroidStopSer, child: Text("停止录屏服务"))
+                ElevatedButton(onPressed:_toAndroidStopSer, child: Text("停止录屏服务")),
+                ElevatedButton(onPressed:_toAndroidCheckInput, child: Text("检查输入权限")),
               ]),
         ));
   }
@@ -116,6 +117,10 @@ class _HomePageState extends State<HomePage> {
     bool res = await toAndroidChannel.invokeMethod("stopSer");
     debugPrint("_toAndroidStopSer:$res");
   }
+  Future<Null> _toAndroidCheckInput() async{
+    bool res = await toAndroidChannel.invokeMethod("checkInput");
+    debugPrint("_toAndroidStopSer:$res");
+}
 
   void onConnect() {
     var id = _idController.text.trim();