improve android server performance

android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt

@@ -17,7 +17,6 @@ import android.graphics.PixelFormat
 import android.hardware.display.DisplayManager.VIRTUAL_DISPLAY_FLAG_PUBLIC
 import android.hardware.display.VirtualDisplay
 import android.media.*
-import android.media.AudioRecord.READ_BLOCKING
 import android.media.projection.MediaProjection
 import android.media.projection.MediaProjectionManager
 import android.os.*
@@ -33,6 +32,7 @@ import java.util.concurrent.Executors
 import kotlin.concurrent.thread
 import org.json.JSONException
 import org.json.JSONObject
+import java.nio.ByteBuffer
 
 const val EXTRA_MP_DATA = "mp_intent"
 const val INIT_SERVICE = "init_service"
@@ -65,33 +65,6 @@ class MainService : Service() {
         System.loadLibrary("rustdesk")
     }
 
-    // rust call jvm
-    @Keep
-    fun rustGetVideoRaw(): ByteArray {
-        return if (videoData != null) {
-            videoData!!
-        } else {
-            videoZeroData
-        }
-    }
-
-    @Keep
-    fun rustGetAudioRaw(): FloatArray {
-        return if (isNewData && audioData != null) {
-            isNewData = false
-            audioData!!
-        } else {
-            audioZeroData
-        }
-    }
-
-    @Keep
-    fun rustGetAudioRawLen(): Int {
-        return if (isNewData && audioData != null && audioData!!.isNotEmpty()) {
-            audioData!!.size
-        } else 0
-    }
-
     @Keep
     fun rustGetByName(name: String): String {
         return when (name) {
@@ -145,13 +118,16 @@ class MainService : Service() {
                 Log.d(logTag, "from rust:stop_capture")
                 stopCapture()
             }
-            else -> {}
+            else -> {
+            }
         }
     }
 
     // jvm call rust
     private external fun init(ctx: Context)
     private external fun startServer()
+    private external fun onVideoFrameUpdate(buf: ByteBuffer)
+    private external fun onAudioFrameUpdate(buf: ByteBuffer)
     private external fun translateLocale(localeName: String, input: String): String
     // private external fun sendVp9(data: ByteArray)
 
@@ -170,21 +146,18 @@ class MainService : Service() {
     val isStart: Boolean
         get() = _isStart
 
+    // video
     private var mediaProjection: MediaProjection? = null
     private var surface: Surface? = null
     private val sendVP9Thread = Executors.newSingleThreadExecutor()
     private var videoEncoder: MediaCodec? = null
-    private var videoData: ByteArray? = null
     private var imageReader: ImageReader? = null
-    private val videoZeroData = ByteArray(32)
     private var virtualDisplay: VirtualDisplay? = null
 
     // audio
     private var audioRecorder: AudioRecord? = null
-    private var audioData: FloatArray? = null
+    private var audioReader: AudioReader? = null
     private var minBufferSize = 0
-    private var isNewData = false
-    private val audioZeroData: FloatArray = FloatArray(32)  
     private var audioRecordStat = false
 
     // notification
@@ -245,7 +218,7 @@ class MainService : Service() {
                     INFO.screenWidth,
                     INFO.screenHeight,
                     PixelFormat.RGBA_8888,
-                    2
+                    4
                 ).apply {
                     setOnImageAvailableListener({ imageReader: ImageReader ->
                         try {
@@ -254,20 +227,10 @@ class MainService : Service() {
                                 val planes = image.planes
                                 val buffer = planes[0].buffer
                                 buffer.rewind()
-                                // Be careful about OOM!
+                                onVideoFrameUpdate(buffer)
-                                if (videoData == null) {
-                                    videoData = ByteArray(buffer.capacity())
-                                    buffer.get(videoData!!)
-                                    Log.d(logTag, "init video ${videoData!!.size}")
-                                } else {
-                                    buffer.get(videoData!!)
-                                }
                             }
                         } catch (ignored: java.lang.Exception) {
                         }
-                        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
-                            imageReader.discardFreeBuffers()
-                        }
                     }, null)
                 }
             Log.d(logTag, "ImageReader.setOnImageAvailableListener done")
@@ -311,7 +274,6 @@ class MainService : Service() {
         }
         virtualDisplay = null
         videoEncoder = null
-        videoData = null
 
         // release audio
         audioRecordStat = false
@@ -370,7 +332,7 @@ class MainService : Service() {
             it.setCallback(cb)
             it.start()
             virtualDisplay = mp.createVirtualDisplay(
-                "rustdesk test",
+                "RustDeskVD",
                 INFO.screenWidth, INFO.screenHeight, 200, VIRTUAL_DISPLAY_FLAG_PUBLIC,
                 surface, null, null
             )
@@ -423,14 +385,13 @@ class MainService : Service() {
     @RequiresApi(Build.VERSION_CODES.M)
     private fun startAudioRecorder() {
         checkAudioRecorder()
-        if (audioData != null && audioRecorder != null && minBufferSize != 0) {
+        if (audioReader != null && audioRecorder != null && minBufferSize != 0) {
             audioRecorder!!.startRecording()
             audioRecordStat = true
             thread {
                 while (audioRecordStat) {
-                    val res = audioRecorder!!.read(audioData!!, 0, minBufferSize, READ_BLOCKING)
+                    audioReader!!.readSync(audioRecorder!!)?.let {
-                    if (res != AudioRecord.ERROR_INVALID_OPERATION) {
+                        onAudioFrameUpdate(it)
-                        isNewData = true
                     }
                 }
                 Log.d(logTag, "Exit audio thread")
@@ -442,10 +403,11 @@ class MainService : Service() {
 
     @RequiresApi(Build.VERSION_CODES.M)
     private fun checkAudioRecorder() {
-        if (audioData != null && audioRecorder != null && minBufferSize != 0) {
+        if (audioRecorder != null && audioRecorder != null && minBufferSize != 0) {
             return
         }
-        minBufferSize = 2 * AudioRecord.getMinBufferSize(
+        // read f32 to byte , length * 4
+        minBufferSize = 2 * 4 * AudioRecord.getMinBufferSize(
             AUDIO_SAMPLE_RATE,
             AUDIO_CHANNEL_MASK,
             AUDIO_ENCODING
@@ -454,8 +416,8 @@ class MainService : Service() {
             Log.d(logTag, "get min buffer size fail!")
             return
         }
-        audioData = FloatArray(minBufferSize)
+        audioReader = AudioReader(minBufferSize, 4)
-        Log.d(logTag, "init audioData len:${audioData!!.size}")
+        Log.d(logTag, "init audioData len:$minBufferSize")
         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
             mediaProjection?.let {
                 val apcc = AudioPlaybackCaptureConfiguration.Builder(it)
@@ -536,7 +498,12 @@ class MainService : Service() {
         startForeground(DEFAULT_NOTIFY_ID, notification)
     }
 
-    private fun loginRequestNotification(clientID:Int, type: String, username: String, peerId: String) {
+    private fun loginRequestNotification(
+        clientID: Int,
+        type: String,
+        username: String,
+        peerId: String
+    ) {
         cancelNotification(clientID)
         val notification = notificationBuilder
             .setOngoing(false)
@@ -550,7 +517,12 @@ class MainService : Service() {
         notificationManager.notify(getClientNotifyID(clientID), notification)
     }
 
-    private fun onClientAuthorizedNotification(clientID: Int, type: String, username: String, peerId: String) {
+    private fun onClientAuthorizedNotification(
+        clientID: Int,
+        type: String,
+        username: String,
+        peerId: String
+    ) {
         cancelNotification(clientID)
         val notification = notificationBuilder
             .setOngoing(false)

android/app/src/main/kotlin/com/carriez/flutter_hbb/common.kt

@@ -2,6 +2,8 @@ package com.carriez.flutter_hbb
 
 import android.annotation.SuppressLint
 import android.content.Context
+import android.media.AudioRecord
+import android.media.AudioRecord.READ_BLOCKING
 import android.media.MediaCodecList
 import android.media.MediaFormat
 import android.os.Build
@@ -11,6 +13,7 @@ import android.util.Log
 import androidx.annotation.RequiresApi
 import com.hjq.permissions.Permission
 import com.hjq.permissions.XXPermissions
+import java.nio.ByteBuffer
 import java.util.*
 
 @SuppressLint("ConstantLocale")
@@ -78,3 +81,39 @@ fun checkPermission(context: Context,type: String): Boolean {
     }
     return XXPermissions.isGranted(context, permission)
 }
+
+class AudioReader(val bufSize: Int, private val maxFrames: Int) {
+    private var currentPos = 0
+    private val bufferPool: Array<ByteBuffer>
+
+    init {
+        if (maxFrames < 0 || maxFrames > 32) {
+            throw Exception("Out of bounds")
+        }
+        if (bufSize <= 0) {
+            throw Exception("Wrong bufSize")
+        }
+        bufferPool = Array(maxFrames) {
+            ByteBuffer.allocateDirect(bufSize)
+        }
+    }
+
+    private fun next() {
+        currentPos++
+        if (currentPos >= maxFrames) {
+            currentPos = 0
+        }
+    }
+
+    @RequiresApi(Build.VERSION_CODES.M)
+    fun readSync(audioRecord: AudioRecord): ByteBuffer? {
+        val buffer = bufferPool[currentPos]
+        val res = audioRecord.read(buffer, bufSize, READ_BLOCKING)
+        return if (res > 0) {
+            next()
+            buffer
+        } else {
+            null
+        }
+    }
+}

android/app/src/main/res/values/strings.xml

@@ -1,4 +1,4 @@
 <resources>
     <string name="app_name">RustDesk</string>
-    <string name="accessibility_service_description">测试服务 输入服务</string>
+    <string name="accessibility_service_description">Allow other devices to control your phone using virtual touch, when RustDesk screen sharing is established</string>
 </resources>

lib/models/server_model.dart

@@ -41,14 +41,14 @@ class ServerModel with ChangeNotifier {
       /**
        * 1. check android permission
        * 2. check config
-       * audio true by default (if permission on)
+       * audio true by default (if permission on) (false default < Android 10)
        * file true by default (if permission on)
        * input false by default (it need turning on manually everytime)
        */
       await Future.delayed(Duration(seconds: 1));
 
       // audio
-      if(!await PermissionManager.check("audio")){
+      if(androidVersion<30 || !await PermissionManager.check("audio")){
         _audioOk = false;
         FFI.setByName('option', jsonEncode(
             Map()