From 06c7bc137fec17e4f6c1abc6fb9537d7af44b6c9 Mon Sep 17 00:00:00 2001 From: 21pages Date: Thu, 14 Nov 2024 21:01:41 +0800 Subject: [PATCH] linux android use cpal (#9914) Signed-off-by: 21pages --- Cargo.toml | 2 - build.rs | 3 +- .../com/carriez/flutter_hbb/MainActivity.kt | 18 +++ flutter/android/app/src/main/kotlin/ffi.kt | 1 + libs/scrap/src/android/ffi.rs | 58 ++++--- res/vcpkg/oboe-wrapper/CMakeLists.txt | 15 -- res/vcpkg/oboe-wrapper/oboe.cc | 118 -------------- res/vcpkg/oboe-wrapper/portfile.cmake | 8 - res/vcpkg/oboe-wrapper/vcpkg.json | 19 --- src/client.rs | 149 +----------------- vcpkg.json | 19 ++- 11 files changed, 79 insertions(+), 331 deletions(-) delete mode 100644 res/vcpkg/oboe-wrapper/CMakeLists.txt delete mode 100644 res/vcpkg/oboe-wrapper/oboe.cc delete mode 100644 res/vcpkg/oboe-wrapper/portfile.cmake delete mode 100644 res/vcpkg/oboe-wrapper/vcpkg.json diff --git a/Cargo.toml b/Cargo.toml index a7788372b..eca0fc55c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,8 +77,6 @@ fon = "0.6" zip = "0.6" shutdown_hooks = "0.1" totp-rs = { version = "5.4", default-features = false, features = ["gen_secret", "otpauth"] } - -[target.'cfg(not(any(target_os = "android", target_os = "linux")))'.dependencies] cpal = "0.15" ringbuf = "0.3" diff --git a/build.rs b/build.rs index d332a43a2..7333fff7b 100644 --- a/build.rs +++ b/build.rs @@ -1,7 +1,7 @@ #[cfg(windows)] fn build_windows() { let file = "src/platform/windows.cc"; - let file2 = "src/platform/windows_delete_test_cert.cc"; + let file2 = "src/platform/windows_delete_test_cert.cc"; cc::Build::new().file(file).file(file2).compile("windows"); println!("cargo:rustc-link-lib=WtsApi32"); println!("cargo:rerun-if-changed={}", file); @@ -72,7 +72,6 @@ fn install_android_deps() { ); println!("cargo:rustc-link-lib=ndk_compat"); println!("cargo:rustc-link-lib=oboe"); - println!("cargo:rustc-link-lib=oboe_wrapper"); println!("cargo:rustc-link-lib=c++"); println!("cargo:rustc-link-lib=OpenSLES"); } diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt index 10c3d7c2d..6b197a39f 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt @@ -30,6 +30,10 @@ import com.hjq.permissions.XXPermissions import io.flutter.embedding.android.FlutterActivity import io.flutter.embedding.engine.FlutterEngine import io.flutter.plugin.common.MethodChannel +import io.flutter.embedding.engine.plugins.FlutterPlugin +import io.flutter.plugin.common.MethodCall +import io.flutter.plugin.common.MethodChannel.MethodCallHandler +import io.flutter.plugin.common.MethodChannel.Result import kotlin.concurrent.thread @@ -57,6 +61,7 @@ class MainActivity : FlutterActivity() { channelTag ) initFlutterChannel(flutterMethodChannel!!) + flutterEngine.plugins.add(ContextPlugin()) thread { setCodecInfo() } } @@ -389,3 +394,16 @@ class MainActivity : FlutterActivity() { stopService(Intent(this, FloatingWindowService::class.java)) } } + +// https://cjycode.com/flutter_rust_bridge/guides/how-to/ndk-init +class ContextPlugin : FlutterPlugin, MethodCallHandler { + override fun onAttachedToEngine(flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { + FFI.initContext(flutterPluginBinding.applicationContext) + } + override fun onMethodCall(call: MethodCall, result: Result) { + result.notImplemented() + } + + override fun onDetachedFromEngine(binding: FlutterPlugin.FlutterPluginBinding) { + } +} \ No newline at end of file diff --git a/flutter/android/app/src/main/kotlin/ffi.kt b/flutter/android/app/src/main/kotlin/ffi.kt index a7573bbf9..43368c1e6 100644 --- a/flutter/android/app/src/main/kotlin/ffi.kt +++ b/flutter/android/app/src/main/kotlin/ffi.kt @@ -11,6 +11,7 @@ object FFI { } external fun init(ctx: Context) + external fun initContext(ctx: Context) external fun startServer(app_dir: String, custom_client_config: String) external fun startService() external fun onVideoFrameUpdate(buf: ByteBuffer) diff --git a/libs/scrap/src/android/ffi.rs b/libs/scrap/src/android/ffi.rs index f5208a673..4608597ce 100644 --- a/libs/scrap/src/android/ffi.rs +++ b/libs/scrap/src/android/ffi.rs @@ -12,6 +12,7 @@ use jni::errors::{Error as JniError, Result as JniResult}; use lazy_static::lazy_static; use serde::Deserialize; use std::ops::Not; +use std::os::raw::c_void; use std::sync::atomic::{AtomicPtr, Ordering::SeqCst}; use std::sync::{Mutex, RwLock}; use std::time::{Duration, Instant}; @@ -155,10 +156,24 @@ pub extern "system" fn Java_ffi_FFI_setFrameRawEnable( pub extern "system" fn Java_ffi_FFI_init(env: JNIEnv, _class: JClass, ctx: JObject) { log::debug!("MainService init from java"); if let Ok(jvm) = env.get_java_vm() { + let java_vm = jvm.get_java_vm_pointer() as *mut c_void; *JVM.write().unwrap() = Some(jvm); if let Ok(context) = env.new_global_ref(ctx) { + let context_jobject = context.as_obj().as_raw() as *mut c_void; *MAIN_SERVICE_CTX.write().unwrap() = Some(context); - init_ndk_context().ok(); + init_ndk_context(java_vm, context_jobject); + } + } +} + +#[no_mangle] +pub extern "system" fn Java_ffi_FFI_initContext(env: JNIEnv, _class: JClass, ctx: JObject) { + log::debug!("MainActivity initContext from java"); + if let Ok(jvm) = env.get_java_vm() { + if let Ok(context) = env.new_global_ref(ctx) { + let java_vm = jvm.get_java_vm_pointer() as *mut c_void; + let context_jobject = context.as_obj().as_raw() as *mut c_void; + init_ndk_context(java_vm, context_jobject); } } } @@ -332,7 +347,14 @@ pub fn call_main_service_set_by_name( } } -fn init_ndk_context() -> JniResult<()> { +// Difference between MainService, MainActivity, JNI_OnLoad: +// jvm is the same, ctx is differen and ctx of JNI_OnLoad is null. +// cpal: all three works +// Service(GetByName, ...): only ctx from MainService works, so use 2 init context functions +// On app start: JNI_OnLoad or MainActivity init context +// On service start first time: MainService replace the context + +fn init_ndk_context(java_vm: *mut c_void, context_jobject: *mut c_void) { let mut lock = NDK_CONTEXT_INITED.lock().unwrap(); if *lock { unsafe { @@ -340,22 +362,20 @@ fn init_ndk_context() -> JniResult<()> { } *lock = false; } - if let (Some(jvm), Some(ctx)) = ( - JVM.read().unwrap().as_ref(), - MAIN_SERVICE_CTX.read().unwrap().as_ref(), - ) { - unsafe { - ndk_context::initialize_android_context( - jvm.get_java_vm_pointer() as _, - ctx.as_obj().as_raw() as _, - ); - #[cfg(feature = "hwcodec")] - hwcodec::android::ffmpeg_set_java_vm( - jvm.get_java_vm_pointer() as _, - ); - } - *lock = true; - return Ok(()); + unsafe { + ndk_context::initialize_android_context(java_vm, context_jobject); + #[cfg(feature = "hwcodec")] + hwcodec::android::ffmpeg_set_java_vm(java_vm); } - Err(JniError::ThrowFailed(-1)) + *lock = true; } + +// // https://cjycode.com/flutter_rust_bridge/guides/how-to/ndk-init +// #[no_mangle] +// pub extern "C" fn JNI_OnLoad(vm: jni::JavaVM, res: *mut std::os::raw::c_void) -> jni::sys::jint { +// if let Ok(env) = vm.get_env() { +// let vm = vm.get_java_vm_pointer() as *mut std::os::raw::c_void; +// init_ndk_context(vm, res); +// } +// jni::JNIVersion::V6.into() +// } diff --git a/res/vcpkg/oboe-wrapper/CMakeLists.txt b/res/vcpkg/oboe-wrapper/CMakeLists.txt deleted file mode 100644 index 9d50a9894..000000000 --- a/res/vcpkg/oboe-wrapper/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -cmake_minimum_required(VERSION 3.20) -project(oboe_wrapper CXX) - -include(GNUInstallDirs) - -add_library(oboe_wrapper STATIC - oboe.cc -) - -target_include_directories(oboe_wrapper PRIVATE "${CURRENT_INSTALLED_DIR}/include") - -install(TARGETS oboe_wrapper - ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" - LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" - RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") diff --git a/res/vcpkg/oboe-wrapper/oboe.cc b/res/vcpkg/oboe-wrapper/oboe.cc deleted file mode 100644 index a3c8238a7..000000000 --- a/res/vcpkg/oboe-wrapper/oboe.cc +++ /dev/null @@ -1,118 +0,0 @@ -#include -#include -#include -#include - -// I got link problem with std::mutex, so use pthread instead -class CThreadLock -{ -public: - CThreadLock(); - virtual ~CThreadLock(); - - void Lock(); - void Unlock(); - -private: - pthread_mutex_t mutexlock; -}; - -CThreadLock::CThreadLock() -{ - // init lock here - pthread_mutex_init(&mutexlock, 0); -} - -CThreadLock::~CThreadLock() -{ - // deinit lock here - pthread_mutex_destroy(&mutexlock); -} -void CThreadLock::Lock() -{ - // lock - pthread_mutex_lock(&mutexlock); -} -void CThreadLock::Unlock() -{ - // unlock - pthread_mutex_unlock(&mutexlock); -} - -class Player : public oboe::AudioStreamDataCallback -{ -public: - Player(int channels, int sample_rate) - { - this->channels = channels; - oboe::AudioStreamBuilder builder; - // The builder set methods can be chained for convenience. - builder.setSharingMode(oboe::SharingMode::Exclusive) - ->setPerformanceMode(oboe::PerformanceMode::LowLatency) - ->setChannelCount(channels) - ->setSampleRate(sample_rate) - ->setFormat(oboe::AudioFormat::Float) - ->setDataCallback(this) - ->openManagedStream(outStream); - // Typically, start the stream after querying some stream information, as well as some input from the user - outStream->requestStart(); - } - - ~Player() { - outStream->requestStop(); - } - - oboe::DataCallbackResult onAudioReady(oboe::AudioStream *oboeStream, void *audioData, int32_t numFrames) override - { - float *floatData = (float *)audioData; - int i = 0; - mtx.Lock(); - auto n = channels * numFrames; - for (; i < n && i < (int)buffer.size(); ++i, ++floatData) - { - *floatData = buffer.front(); - buffer.pop_front(); - } - mtx.Unlock(); - for (; i < n; ++i, ++floatData) - { - *floatData = 0; - } - return oboe::DataCallbackResult::Continue; - } - - void push(const float *v, int n) - { - mtx.Lock(); - for (auto i = 0; i < n; ++i, ++v) - buffer.push_back(*v); - // in case memory overuse - if (buffer.size() > 48 * 1024 * 120) - buffer.clear(); - mtx.Unlock(); - } - -private: - oboe::ManagedStream outStream; - int channels; - std::deque buffer; - CThreadLock mtx; -}; - -extern "C" -{ - void *create_oboe_player(int channels, int sample_rate) - { - return new Player(channels, sample_rate); - } - - void push_oboe_data(void *player, const float* v, int n) - { - static_cast(player)->push(v, n); - } - - void destroy_oboe_player(void *player) - { - delete static_cast(player); - } -} \ No newline at end of file diff --git a/res/vcpkg/oboe-wrapper/portfile.cmake b/res/vcpkg/oboe-wrapper/portfile.cmake deleted file mode 100644 index c83f5bcb1..000000000 --- a/res/vcpkg/oboe-wrapper/portfile.cmake +++ /dev/null @@ -1,8 +0,0 @@ -vcpkg_configure_cmake( - SOURCE_PATH "${CMAKE_CURRENT_LIST_DIR}" - OPTIONS - -DCURRENT_INSTALLED_DIR=${CURRENT_INSTALLED_DIR} - PREFER_NINJA -) - -vcpkg_cmake_install() diff --git a/res/vcpkg/oboe-wrapper/vcpkg.json b/res/vcpkg/oboe-wrapper/vcpkg.json deleted file mode 100644 index be497e1bb..000000000 --- a/res/vcpkg/oboe-wrapper/vcpkg.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "oboe-wrapper", - "version": "0", - "description": "None", - "dependencies": [ - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - }, - { - "name": "oboe", - "host": false - } - ] -} diff --git a/src/client.rs b/src/client.rs index f7453b88d..2abc4d0ff 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2,14 +2,12 @@ use async_trait::async_trait; use bytes::Bytes; #[cfg(not(any(target_os = "android", target_os = "ios")))] use clipboard_master::{CallbackResult, ClipboardHandler}; -#[cfg(not(any(target_os = "android", target_os = "linux")))] use cpal::{ traits::{DeviceTrait, HostTrait, StreamTrait}, Device, Host, StreamConfig, }; use crossbeam_queue::ArrayQueue; use magnum_opus::{Channels::*, Decoder as AudioDecoder}; -#[cfg(not(any(target_os = "android", target_os = "linux")))] use ringbuf::{ring_buffer::RbBase, Rb}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -117,7 +115,6 @@ pub const SCRAP_OTHER_VERSION_OR_X11_REQUIRED: &str = pub const SCRAP_X11_REQUIRED: &str = "x11 expected"; pub const SCRAP_X11_REF_URL: &str = "https://rustdesk.com/docs/en/manual/linux/#x11-required"; -#[cfg(not(any(target_os = "android", target_os = "linux")))] pub const AUDIO_BUFFER_MS: usize = 3000; #[cfg(feature = "flutter")] @@ -140,7 +137,6 @@ struct TextClipboardState { running: bool, } -#[cfg(not(any(target_os = "android", target_os = "linux")))] lazy_static::lazy_static! { static ref AUDIO_HOST: Host = cpal::default_host(); } @@ -163,66 +159,6 @@ pub fn get_key_state(key: enigo::Key) -> bool { ENIGO.lock().unwrap().get_key_state(key) } -cfg_if::cfg_if! { - if #[cfg(target_os = "android")] { - -use hbb_common::libc::{c_float, c_int}; -type Oboe = *mut c_void; -extern "C" { - fn create_oboe_player(channels: c_int, sample_rate: c_int) -> Oboe; - fn push_oboe_data(oboe: Oboe, d: *const c_float, n: c_int); - fn destroy_oboe_player(oboe: Oboe); -} - -struct OboePlayer { - raw: Oboe, -} - -impl Default for OboePlayer { - fn default() -> Self { - Self { - raw: std::ptr::null_mut(), - } - } -} - -impl OboePlayer { - fn new(channels: i32, sample_rate: i32) -> Self { - unsafe { - Self { - raw: create_oboe_player(channels, sample_rate), - } - } - } - - #[cfg(not(any(target_os = "android", target_os = "ios")))] - fn is_null(&self) -> bool { - self.raw.is_null() - } - - fn push(&mut self, d: &[f32]) { - if self.raw.is_null() { - return; - } - unsafe { - push_oboe_data(self.raw, d.as_ptr(), d.len() as _); - } - } -} - -impl Drop for OboePlayer { - fn drop(&mut self) { - unsafe { - if !self.raw.is_null() { - destroy_oboe_player(self.raw); - } - } - } -} - -} -} - impl Client { /// Start a new connection. pub async fn start( @@ -887,30 +823,20 @@ impl ClipboardHandler for ClientClipboardHandler { #[derive(Default)] pub struct AudioHandler { audio_decoder: Option<(AudioDecoder, Vec)>, - #[cfg(target_os = "android")] - oboe: Option, - #[cfg(target_os = "linux")] - simple: Option, - #[cfg(not(any(target_os = "android", target_os = "linux")))] audio_buffer: AudioBuffer, sample_rate: (u32, u32), - #[cfg(not(any(target_os = "android", target_os = "linux")))] audio_stream: Option>, channels: u16, - #[cfg(not(any(target_os = "android", target_os = "linux")))] device_channel: u16, - #[cfg(not(any(target_os = "android", target_os = "linux")))] ready: Arc>, } -#[cfg(not(any(target_os = "android", target_os = "linux")))] struct AudioBuffer( pub Arc>>, usize, [usize; 30], ); -#[cfg(not(any(target_os = "android", target_os = "linux")))] impl Default for AudioBuffer { fn default() -> Self { Self( @@ -923,7 +849,6 @@ impl Default for AudioBuffer { } } -#[cfg(not(any(target_os = "android", target_os = "linux")))] impl AudioBuffer { pub fn resize(&mut self, sample_rate: usize, channels: usize) { let capacity = sample_rate * channels * AUDIO_BUFFER_MS / 1000; @@ -1026,48 +951,6 @@ impl AudioBuffer { impl AudioHandler { /// Start the audio playback. - #[cfg(target_os = "linux")] - fn start_audio(&mut self, format0: AudioFormat) -> ResultType<()> { - use psimple::Simple; - use pulse::sample::{Format, Spec}; - use pulse::stream::Direction; - - let spec = Spec { - format: Format::F32le, - channels: format0.channels as _, - rate: format0.sample_rate as _, - }; - if !spec.is_valid() { - bail!("Invalid audio format"); - } - - self.simple = Some(Simple::new( - None, // Use the default server - &crate::get_app_name(), // Our application’s name - Direction::Playback, // We want a playback stream - None, // Use the default device - "playback", // Description of our stream - &spec, // Our sample format - None, // Use default channel map - None, // Use default buffering attributes - )?); - self.sample_rate = (format0.sample_rate, format0.sample_rate); - Ok(()) - } - - /// Start the audio playback. - #[cfg(target_os = "android")] - fn start_audio(&mut self, format0: AudioFormat) -> ResultType<()> { - self.oboe = Some(OboePlayer::new( - format0.channels as _, - format0.sample_rate as _, - )); - self.sample_rate = (format0.sample_rate, format0.sample_rate); - Ok(()) - } - - /// Start the audio playback. - #[cfg(not(any(target_os = "android", target_os = "linux")))] fn start_audio(&mut self, format0: AudioFormat) -> ResultType<()> { let device = AUDIO_HOST .default_output_device() @@ -1130,24 +1013,13 @@ impl AudioHandler { /// Handle audio frame and play it. #[inline] pub fn handle_frame(&mut self, frame: AudioFrame) { - #[cfg(not(any(target_os = "android", target_os = "linux")))] if self.audio_stream.is_none() || !self.ready.lock().unwrap().clone() { return; } - #[cfg(target_os = "linux")] - if self.simple.is_none() { - log::debug!("PulseAudio simple binding does not exists"); - return; - } - #[cfg(target_os = "android")] - if self.oboe.is_none() { - return; - } self.audio_decoder.as_mut().map(|(d, buffer)| { if let Ok(n) = d.decode_float(&frame.data, buffer, false) { let channels = self.channels; let n = n * (channels as usize); - #[cfg(not(any(target_os = "android", target_os = "linux")))] { let sample_rate0 = self.sample_rate.0; let sample_rate = self.sample_rate.1; @@ -1171,22 +1043,11 @@ impl AudioHandler { } self.audio_buffer.append_pcm(&buffer); } - #[cfg(target_os = "android")] - { - self.oboe.as_mut().map(|x| x.push(&buffer[0..n])); - } - #[cfg(target_os = "linux")] - { - let data_u8 = - unsafe { std::slice::from_raw_parts::(buffer.as_ptr() as _, n * 4) }; - self.simple.as_mut().map(|x| x.write(data_u8)); - } } }); } /// Build audio output stream for current device. - #[cfg(not(any(target_os = "android", target_os = "linux")))] fn build_output_stream>( &mut self, config: &StreamConfig, @@ -1212,6 +1073,8 @@ impl AudioHandler { let mut n = data.len(); let mut lock = audio_buffer.lock().unwrap(); let mut having = lock.occupied_len(); + // android two timestamps, one from zero, another not + #[cfg(not(target_os = "android"))] if having < n { let tms = info.timestamp(); let how_long = tms @@ -1220,7 +1083,8 @@ impl AudioHandler { .unwrap_or(Duration::from_millis(0)); // must long enough to fight back scheuler delay - if how_long > Duration::from_millis(6) { + if how_long > Duration::from_millis(6) && how_long < Duration::from_millis(3000) + { drop(lock); std::thread::sleep(how_long.div_f32(1.2)); lock = audio_buffer.lock().unwrap(); @@ -1231,7 +1095,10 @@ impl AudioHandler { n = having; } } - + #[cfg(target_os = "android")] + if having < n { + n = having; + } let mut elems = vec![0.0f32; n]; if n > 0 { lock.pop_slice(&mut elems); diff --git a/vcpkg.json b/vcpkg.json index 81484772a..75cee85b1 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -24,10 +24,6 @@ "name": "oboe", "platform": "android" }, - { - "name": "oboe-wrapper", - "platform": "android" - }, { "name": "opus", "host": true @@ -87,8 +83,17 @@ ] }, "overrides": [ - { "name": "ffnvcodec", "version": "12.1.14.0" }, - { "name": "amd-amf", "version": "1.4.29" }, - { "name": "mfx-dispatch", "version": "1.35.1" } + { + "name": "ffnvcodec", + "version": "12.1.14.0" + }, + { + "name": "amd-amf", + "version": "1.4.29" + }, + { + "name": "mfx-dispatch", + "version": "1.35.1" + } ] } \ No newline at end of file