linux android use cpal (#9914)

Signed-off-by: 21pages <sunboeasy@gmail.com>

Cargo.toml

@@ -77,8 +77,6 @@ fon = "0.6"
 zip = "0.6"
 shutdown_hooks = "0.1"
 totp-rs = { version = "5.4", default-features = false, features = ["gen_secret", "otpauth"] }
-
-[target.'cfg(not(any(target_os = "android", target_os = "linux")))'.dependencies]
 cpal = "0.15"
 ringbuf = "0.3"
 

build.rs

@@ -72,7 +72,6 @@ fn install_android_deps() {
     );
     println!("cargo:rustc-link-lib=ndk_compat");
     println!("cargo:rustc-link-lib=oboe");
-    println!("cargo:rustc-link-lib=oboe_wrapper");
     println!("cargo:rustc-link-lib=c++");
     println!("cargo:rustc-link-lib=OpenSLES");
 }

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt

@@ -30,6 +30,10 @@ import com.hjq.permissions.XXPermissions
 import io.flutter.embedding.android.FlutterActivity
 import io.flutter.embedding.engine.FlutterEngine
 import io.flutter.plugin.common.MethodChannel
+import io.flutter.embedding.engine.plugins.FlutterPlugin
+import io.flutter.plugin.common.MethodCall
+import io.flutter.plugin.common.MethodChannel.MethodCallHandler
+import io.flutter.plugin.common.MethodChannel.Result
 import kotlin.concurrent.thread
 
 
@@ -57,6 +61,7 @@ class MainActivity : FlutterActivity() {
             channelTag
         )
         initFlutterChannel(flutterMethodChannel!!)
+        flutterEngine.plugins.add(ContextPlugin())
         thread { setCodecInfo() }
     }
 
@@ -389,3 +394,16 @@ class MainActivity : FlutterActivity() {
         stopService(Intent(this, FloatingWindowService::class.java))
     }
 }
+
+// https://cjycode.com/flutter_rust_bridge/guides/how-to/ndk-init
+class ContextPlugin : FlutterPlugin, MethodCallHandler {
+    override fun onAttachedToEngine(flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) {
+        FFI.initContext(flutterPluginBinding.applicationContext)
+    }
+    override fun onMethodCall(call: MethodCall, result: Result) {
+        result.notImplemented()
+    }
+
+    override fun onDetachedFromEngine(binding: FlutterPlugin.FlutterPluginBinding) {
+    }
+}

flutter/android/app/src/main/kotlin/ffi.kt

@@ -11,6 +11,7 @@ object FFI {
     }
 
     external fun init(ctx: Context)
+    external fun initContext(ctx: Context)
     external fun startServer(app_dir: String, custom_client_config: String)
     external fun startService()
     external fun onVideoFrameUpdate(buf: ByteBuffer)

libs/scrap/src/android/ffi.rs

@@ -12,6 +12,7 @@ use jni::errors::{Error as JniError, Result as JniResult};
 use lazy_static::lazy_static;
 use serde::Deserialize;
 use std::ops::Not;
+use std::os::raw::c_void;
 use std::sync::atomic::{AtomicPtr, Ordering::SeqCst};
 use std::sync::{Mutex, RwLock};
 use std::time::{Duration, Instant};
@@ -155,10 +156,24 @@ pub extern "system" fn Java_ffi_FFI_setFrameRawEnable(
 pub extern "system" fn Java_ffi_FFI_init(env: JNIEnv, _class: JClass, ctx: JObject) {
     log::debug!("MainService init from java");
     if let Ok(jvm) = env.get_java_vm() {
+        let java_vm = jvm.get_java_vm_pointer() as *mut c_void;
         *JVM.write().unwrap() = Some(jvm);
         if let Ok(context) = env.new_global_ref(ctx) {
+            let context_jobject = context.as_obj().as_raw() as *mut c_void;
             *MAIN_SERVICE_CTX.write().unwrap() = Some(context);
-            init_ndk_context().ok();
+            init_ndk_context(java_vm, context_jobject);
+        }
+    }
+}
+
+#[no_mangle]
+pub extern "system" fn Java_ffi_FFI_initContext(env: JNIEnv, _class: JClass, ctx: JObject) {
+    log::debug!("MainActivity initContext from java");
+    if let Ok(jvm) = env.get_java_vm() {
+        if let Ok(context) = env.new_global_ref(ctx) {
+            let java_vm = jvm.get_java_vm_pointer() as *mut c_void;
+            let context_jobject = context.as_obj().as_raw() as *mut c_void;
+            init_ndk_context(java_vm, context_jobject);
         }
     }
 }
@@ -332,7 +347,14 @@ pub fn call_main_service_set_by_name(
     }
 }
 
-fn init_ndk_context() -> JniResult<()> {
+// Difference between MainService, MainActivity, JNI_OnLoad:
+//  jvm is the same, ctx is differen and ctx of JNI_OnLoad is null.
+//  cpal: all three works
+//  Service(GetByName, ...): only ctx from MainService works, so use 2 init context functions
+// On app start: JNI_OnLoad or MainActivity init context
+// On service start first time: MainService replace the context
+
+fn init_ndk_context(java_vm: *mut c_void, context_jobject: *mut c_void) {
     let mut lock = NDK_CONTEXT_INITED.lock().unwrap();
     if *lock {
         unsafe {
@@ -340,22 +362,20 @@ fn init_ndk_context() -> JniResult<()> {
         }
         *lock = false;
     }
-    if let (Some(jvm), Some(ctx)) = (
-        JVM.read().unwrap().as_ref(),
-        MAIN_SERVICE_CTX.read().unwrap().as_ref(),
-    ) {
     unsafe {
-            ndk_context::initialize_android_context(
+        ndk_context::initialize_android_context(java_vm, context_jobject);
-                jvm.get_java_vm_pointer() as _,
-                ctx.as_obj().as_raw() as _,
-            );
         #[cfg(feature = "hwcodec")]
-            hwcodec::android::ffmpeg_set_java_vm(
+        hwcodec::android::ffmpeg_set_java_vm(java_vm);
-                jvm.get_java_vm_pointer() as _,
-            );
     }
     *lock = true;
-        return Ok(());
-    }
-    Err(JniError::ThrowFailed(-1))
 }
+
+// // https://cjycode.com/flutter_rust_bridge/guides/how-to/ndk-init
+// #[no_mangle]
+// pub extern "C" fn JNI_OnLoad(vm: jni::JavaVM, res: *mut std::os::raw::c_void) -> jni::sys::jint {
+//     if let Ok(env) = vm.get_env() {
+//         let vm = vm.get_java_vm_pointer() as *mut std::os::raw::c_void;
+//         init_ndk_context(vm, res);
+//     }
+//     jni::JNIVersion::V6.into()
+// }

res/vcpkg/oboe-wrapper/CMakeLists.txt

@@ -1,15 +0,0 @@
-cmake_minimum_required(VERSION 3.20)
-project(oboe_wrapper CXX)
-
-include(GNUInstallDirs)
-
-add_library(oboe_wrapper STATIC
-    oboe.cc
-)
-
-target_include_directories(oboe_wrapper PRIVATE "${CURRENT_INSTALLED_DIR}/include")
-
-install(TARGETS oboe_wrapper
-        ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-        LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-        RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")

res/vcpkg/oboe-wrapper/oboe.cc

@@ -1,118 +0,0 @@
-#include <oboe/Oboe.h>
-#include <math.h>
-#include <deque>
-#include <pthread.h>
-
-// I got link problem with std::mutex, so use pthread instead
-class CThreadLock
-{
-public:
-    CThreadLock();
-    virtual ~CThreadLock();
-
-    void Lock();
-    void Unlock();
-
-private:
-    pthread_mutex_t mutexlock;
-};
-
-CThreadLock::CThreadLock()
-{
-    // init lock here
-    pthread_mutex_init(&mutexlock, 0);
-}
-
-CThreadLock::~CThreadLock()
-{
-    // deinit lock here
-    pthread_mutex_destroy(&mutexlock);
-}
-void CThreadLock::Lock()
-{
-    // lock
-    pthread_mutex_lock(&mutexlock);
-}
-void CThreadLock::Unlock()
-{
-    // unlock
-    pthread_mutex_unlock(&mutexlock);
-}
-
-class Player : public oboe::AudioStreamDataCallback
-{
-public:
-    Player(int channels, int sample_rate)
-    {
-        this->channels = channels;
-        oboe::AudioStreamBuilder builder;
-        // The builder set methods can be chained for convenience.
-        builder.setSharingMode(oboe::SharingMode::Exclusive)
-            ->setPerformanceMode(oboe::PerformanceMode::LowLatency)
-            ->setChannelCount(channels)
-            ->setSampleRate(sample_rate)
-            ->setFormat(oboe::AudioFormat::Float)
-            ->setDataCallback(this)
-            ->openManagedStream(outStream);
-        // Typically, start the stream after querying some stream information, as well as some input from the user
-        outStream->requestStart();
-    }
-
-    ~Player() {
-        outStream->requestStop();
-    }
-
-    oboe::DataCallbackResult onAudioReady(oboe::AudioStream *oboeStream, void *audioData, int32_t numFrames) override
-    {
-        float *floatData = (float *)audioData;
-        int i = 0;
-        mtx.Lock();
-        auto n = channels * numFrames;
-        for (; i < n && i < (int)buffer.size(); ++i, ++floatData)
-        {
-            *floatData = buffer.front();
-            buffer.pop_front();
-        }
-        mtx.Unlock();
-        for (; i < n; ++i, ++floatData)
-        {
-            *floatData = 0;
-        }
-        return oboe::DataCallbackResult::Continue;
-    }
-
-    void push(const float *v, int n)
-    {
-        mtx.Lock();
-        for (auto i = 0; i < n; ++i, ++v)
-            buffer.push_back(*v);
-        // in case memory overuse
-        if (buffer.size() > 48 * 1024 * 120)
-            buffer.clear();
-        mtx.Unlock();
-    }
-
-private:
-    oboe::ManagedStream outStream;
-    int channels;
-    std::deque<float> buffer;
-    CThreadLock mtx;
-};
-
-extern "C"
-{
-    void *create_oboe_player(int channels, int sample_rate)
-    {
-        return new Player(channels, sample_rate);
-    }
-
-    void push_oboe_data(void *player, const float* v, int n)
-    {
-        static_cast<Player *>(player)->push(v, n);
-    }
-
-    void destroy_oboe_player(void *player)
-    {
-        delete static_cast<Player *>(player);
-    }
-}

res/vcpkg/oboe-wrapper/portfile.cmake

@@ -1,8 +0,0 @@
-vcpkg_configure_cmake(
-    SOURCE_PATH "${CMAKE_CURRENT_LIST_DIR}"
-    OPTIONS
-    -DCURRENT_INSTALLED_DIR=${CURRENT_INSTALLED_DIR}
-    PREFER_NINJA
-)
-
-vcpkg_cmake_install()

res/vcpkg/oboe-wrapper/vcpkg.json

@@ -1,19 +0,0 @@
-{
-  "name": "oboe-wrapper",
-  "version": "0",
-  "description": "None",
-  "dependencies": [
-    {
-      "name": "vcpkg-cmake",
-      "host": true
-    },
-    {
-      "name": "vcpkg-cmake-config",
-      "host": true
-    },
-    {
-      "name": "oboe",
-      "host": false
-    }
-  ]
-}

src/client.rs

@@ -2,14 +2,12 @@ use async_trait::async_trait;
 use bytes::Bytes;
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
 use clipboard_master::{CallbackResult, ClipboardHandler};
-#[cfg(not(any(target_os = "android", target_os = "linux")))]
 use cpal::{
     traits::{DeviceTrait, HostTrait, StreamTrait},
     Device, Host, StreamConfig,
 };
 use crossbeam_queue::ArrayQueue;
 use magnum_opus::{Channels::*, Decoder as AudioDecoder};
-#[cfg(not(any(target_os = "android", target_os = "linux")))]
 use ringbuf::{ring_buffer::RbBase, Rb};
 use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
@@ -117,7 +115,6 @@ pub const SCRAP_OTHER_VERSION_OR_X11_REQUIRED: &str =
 pub const SCRAP_X11_REQUIRED: &str = "x11 expected";
 pub const SCRAP_X11_REF_URL: &str = "https://rustdesk.com/docs/en/manual/linux/#x11-required";
 
-#[cfg(not(any(target_os = "android", target_os = "linux")))]
 pub const AUDIO_BUFFER_MS: usize = 3000;
 
 #[cfg(feature = "flutter")]
@@ -140,7 +137,6 @@ struct TextClipboardState {
     running: bool,
 }
 
-#[cfg(not(any(target_os = "android", target_os = "linux")))]
 lazy_static::lazy_static! {
     static ref AUDIO_HOST: Host = cpal::default_host();
 }
@@ -163,66 +159,6 @@ pub fn get_key_state(key: enigo::Key) -> bool {
     ENIGO.lock().unwrap().get_key_state(key)
 }
 
-cfg_if::cfg_if! {
-    if #[cfg(target_os = "android")] {
-
-use hbb_common::libc::{c_float, c_int};
-type Oboe = *mut c_void;
-extern "C" {
-    fn create_oboe_player(channels: c_int, sample_rate: c_int) -> Oboe;
-    fn push_oboe_data(oboe: Oboe, d: *const c_float, n: c_int);
-    fn destroy_oboe_player(oboe: Oboe);
-}
-
-struct OboePlayer {
-    raw: Oboe,
-}
-
-impl Default for OboePlayer {
-    fn default() -> Self {
-        Self {
-            raw: std::ptr::null_mut(),
-        }
-    }
-}
-
-impl OboePlayer {
-    fn new(channels: i32, sample_rate: i32) -> Self {
-        unsafe {
-            Self {
-                raw: create_oboe_player(channels, sample_rate),
-            }
-        }
-    }
-
-    #[cfg(not(any(target_os = "android", target_os = "ios")))]
-    fn is_null(&self) -> bool {
-        self.raw.is_null()
-    }
-
-    fn push(&mut self, d: &[f32]) {
-        if self.raw.is_null() {
-            return;
-        }
-        unsafe {
-            push_oboe_data(self.raw, d.as_ptr(), d.len() as _);
-        }
-    }
-}
-
-impl Drop for OboePlayer {
-    fn drop(&mut self) {
-        unsafe {
-            if !self.raw.is_null() {
-                destroy_oboe_player(self.raw);
-            }
-        }
-    }
-}
-
-}
-}
-
 impl Client {
     /// Start a new connection.
     pub async fn start(
@@ -887,30 +823,20 @@ impl ClipboardHandler for ClientClipboardHandler {
 #[derive(Default)]
 pub struct AudioHandler {
     audio_decoder: Option<(AudioDecoder, Vec<f32>)>,
-    #[cfg(target_os = "android")]
-    oboe: Option<OboePlayer>,
-    #[cfg(target_os = "linux")]
-    simple: Option<psimple::Simple>,
-    #[cfg(not(any(target_os = "android", target_os = "linux")))]
     audio_buffer: AudioBuffer,
     sample_rate: (u32, u32),
-    #[cfg(not(any(target_os = "android", target_os = "linux")))]
     audio_stream: Option<Box<dyn StreamTrait>>,
     channels: u16,
-    #[cfg(not(any(target_os = "android", target_os = "linux")))]
     device_channel: u16,
-    #[cfg(not(any(target_os = "android", target_os = "linux")))]
     ready: Arc<std::sync::Mutex<bool>>,
 }
 
-#[cfg(not(any(target_os = "android", target_os = "linux")))]
 struct AudioBuffer(
     pub Arc<std::sync::Mutex<ringbuf::HeapRb<f32>>>,
     usize,
     [usize; 30],
 );
 
-#[cfg(not(any(target_os = "android", target_os = "linux")))]
 impl Default for AudioBuffer {
     fn default() -> Self {
         Self(
@@ -923,7 +849,6 @@ impl Default for AudioBuffer {
     }
 }
 
-#[cfg(not(any(target_os = "android", target_os = "linux")))]
 impl AudioBuffer {
     pub fn resize(&mut self, sample_rate: usize, channels: usize) {
         let capacity = sample_rate * channels * AUDIO_BUFFER_MS / 1000;
@@ -1026,48 +951,6 @@ impl AudioBuffer {
 
 impl AudioHandler {
     /// Start the audio playback.
-    #[cfg(target_os = "linux")]
-    fn start_audio(&mut self, format0: AudioFormat) -> ResultType<()> {
-        use psimple::Simple;
-        use pulse::sample::{Format, Spec};
-        use pulse::stream::Direction;
-
-        let spec = Spec {
-            format: Format::F32le,
-            channels: format0.channels as _,
-            rate: format0.sample_rate as _,
-        };
-        if !spec.is_valid() {
-            bail!("Invalid audio format");
-        }
-
-        self.simple = Some(Simple::new(
-            None,                   // Use the default server
-            &crate::get_app_name(), // Our application’s name
-            Direction::Playback,    // We want a playback stream
-            None,                   // Use the default device
-            "playback",             // Description of our stream
-            &spec,                  // Our sample format
-            None,                   // Use default channel map
-            None,                   // Use default buffering attributes
-        )?);
-        self.sample_rate = (format0.sample_rate, format0.sample_rate);
-        Ok(())
-    }
-
-    /// Start the audio playback.
-    #[cfg(target_os = "android")]
-    fn start_audio(&mut self, format0: AudioFormat) -> ResultType<()> {
-        self.oboe = Some(OboePlayer::new(
-            format0.channels as _,
-            format0.sample_rate as _,
-        ));
-        self.sample_rate = (format0.sample_rate, format0.sample_rate);
-        Ok(())
-    }
-
-    /// Start the audio playback.
-    #[cfg(not(any(target_os = "android", target_os = "linux")))]
     fn start_audio(&mut self, format0: AudioFormat) -> ResultType<()> {
         let device = AUDIO_HOST
             .default_output_device()
@@ -1130,24 +1013,13 @@ impl AudioHandler {
     /// Handle audio frame and play it.
     #[inline]
     pub fn handle_frame(&mut self, frame: AudioFrame) {
-        #[cfg(not(any(target_os = "android", target_os = "linux")))]
         if self.audio_stream.is_none() || !self.ready.lock().unwrap().clone() {
             return;
         }
-        #[cfg(target_os = "linux")]
-        if self.simple.is_none() {
-            log::debug!("PulseAudio simple binding does not exists");
-            return;
-        }
-        #[cfg(target_os = "android")]
-        if self.oboe.is_none() {
-            return;
-        }
         self.audio_decoder.as_mut().map(|(d, buffer)| {
             if let Ok(n) = d.decode_float(&frame.data, buffer, false) {
                 let channels = self.channels;
                 let n = n * (channels as usize);
-                #[cfg(not(any(target_os = "android", target_os = "linux")))]
                 {
                     let sample_rate0 = self.sample_rate.0;
                     let sample_rate = self.sample_rate.1;
@@ -1171,22 +1043,11 @@ impl AudioHandler {
                     }
                     self.audio_buffer.append_pcm(&buffer);
                 }
-                #[cfg(target_os = "android")]
-                {
-                    self.oboe.as_mut().map(|x| x.push(&buffer[0..n]));
-                }
-                #[cfg(target_os = "linux")]
-                {
-                    let data_u8 =
-                        unsafe { std::slice::from_raw_parts::<u8>(buffer.as_ptr() as _, n * 4) };
-                    self.simple.as_mut().map(|x| x.write(data_u8));
-                }
             }
         });
     }
 
     /// Build audio output stream for current device.
-    #[cfg(not(any(target_os = "android", target_os = "linux")))]
     fn build_output_stream<T: cpal::Sample + cpal::SizedSample + cpal::FromSample<f32>>(
         &mut self,
         config: &StreamConfig,
@@ -1212,6 +1073,8 @@ impl AudioHandler {
                 let mut n = data.len();
                 let mut lock = audio_buffer.lock().unwrap();
                 let mut having = lock.occupied_len();
+                // android two timestamps, one from zero, another not
+                #[cfg(not(target_os = "android"))]
                 if having < n {
                     let tms = info.timestamp();
                     let how_long = tms
@@ -1220,7 +1083,8 @@ impl AudioHandler {
                         .unwrap_or(Duration::from_millis(0));
 
                     // must long enough to fight back scheuler delay
-                    if how_long > Duration::from_millis(6) {
+                    if how_long > Duration::from_millis(6) && how_long < Duration::from_millis(3000)
+                    {
                         drop(lock);
                         std::thread::sleep(how_long.div_f32(1.2));
                         lock = audio_buffer.lock().unwrap();
@@ -1231,7 +1095,10 @@ impl AudioHandler {
                         n = having;
                     }
                 }
-
+                #[cfg(target_os = "android")]
+                if having < n {
+                    n = having;
+                }
                 let mut elems = vec![0.0f32; n];
                 if n > 0 {
                     lock.pop_slice(&mut elems);

vcpkg.json

@@ -24,10 +24,6 @@
       "name": "oboe",
       "platform": "android"
     },
-    {
-      "name": "oboe-wrapper",
-      "platform": "android"
-    },
     {
       "name": "opus",
       "host": true
@@ -87,8 +83,17 @@
     ]
   },
   "overrides": [
-    { "name": "ffnvcodec", "version": "12.1.14.0" },
+    {
-    { "name": "amd-amf", "version": "1.4.29" },
+      "name": "ffnvcodec",
-    { "name": "mfx-dispatch", "version": "1.35.1" }
+      "version": "12.1.14.0"
+    },
+    {
+      "name": "amd-amf",
+      "version": "1.4.29"
+    },
+    {
+      "name": "mfx-dispatch",
+      "version": "1.35.1"
+    }
   ]
 }