From 30afe4f779e4263929677d189300992a7e2a0860 Mon Sep 17 00:00:00 2001 From: fufesou <13586388+fufesou@users.noreply.github.com> Date: Sun, 14 Jul 2024 04:07:02 +0800 Subject: [PATCH] refact: seperate audio device for voice call (#8703) Signed-off-by: fufesou --- flutter/lib/common/widgets/audio_input.dart | 45 ++++++++++++++----- .../desktop/pages/desktop_setting_page.dart | 6 ++- flutter/lib/desktop/pages/server_page.dart | 6 +-- .../lib/desktop/widgets/remote_toolbar.dart | 45 ++++++++++--------- src/client/io_loop.rs | 14 +++--- src/flutter_ffi.rs | 19 ++++++++ src/ipc.rs | 18 +++++--- src/server/audio_service.rs | 39 ++++++++++++++-- src/server/connection.rs | 44 +++++++++--------- 9 files changed, 162 insertions(+), 74 deletions(-) diff --git a/flutter/lib/common/widgets/audio_input.dart b/flutter/lib/common/widgets/audio_input.dart index 36a0e4972..1db439127 100644 --- a/flutter/lib/common/widgets/audio_input.dart +++ b/flutter/lib/common/widgets/audio_input.dart @@ -2,22 +2,39 @@ import 'package:flutter/material.dart'; import 'package:flutter_hbb/common.dart'; import 'package:flutter_hbb/models/platform_model.dart'; +const _kWindowsSystemSound = 'System Sound'; + typedef AudioINputSetDevice = void Function(String device); typedef AudioInputBuilder = Widget Function( List devices, String currentDevice, AudioINputSetDevice setDevice); class AudioInput extends StatelessWidget { final AudioInputBuilder builder; + final bool isCm; + final bool isVoiceCall; - const AudioInput({Key? key, required this.builder}) : super(key: key); + const AudioInput( + {Key? key, + required this.builder, + required this.isCm, + required this.isVoiceCall}) + : super(key: key); static String getDefault() { if (isWindows) return translate('System Sound'); return ''; } - static Future getValue() async { - String device = await bind.mainGetOption(key: 'audio-input'); + static Future getAudioInput(bool isCm, bool isVoiceCall) { + if (isVoiceCall) { + return bind.getVoiceCallInputDevice(isCm: isCm); + } else { + return bind.mainGetOption(key: 'audio-input'); + } + } + + static Future getValue(bool isCm, bool isVoiceCall) async { + String device = await getAudioInput(isCm, isVoiceCall); if (device.isNotEmpty) { return device; } else { @@ -25,31 +42,39 @@ class AudioInput extends StatelessWidget { } } - static Future setDevice(String device) async { + static Future setDevice( + String device, bool isCm, bool isVoiceCall) async { if (device == getDefault()) device = ''; - await bind.mainSetOption(key: 'audio-input', value: device); + if (isVoiceCall) { + await bind.setVoiceCallInputDevice(isCm: isCm, device: device); + } else { + await bind.mainSetOption(key: 'audio-input', value: device); + } } - static Future> getDevicesInfo() async { + static Future> getDevicesInfo( + bool isCm, bool isVoiceCall) async { List devices = (await bind.mainGetSoundInputs()).toList(); if (isWindows) { - devices.insert(0, translate('System Sound')); + devices.insert(0, translate(_kWindowsSystemSound)); } - String current = await getValue(); + String current = await getValue(isCm, isVoiceCall); return {'devices': devices, 'current': current}; } @override Widget build(BuildContext context) { return futureBuilder( - future: getDevicesInfo(), + future: getDevicesInfo(isCm, isVoiceCall), hasData: (data) { String currentDevice = data['current']; List devices = data['devices'] as List; if (devices.isEmpty) { return const Offstage(); } - return builder(devices, currentDevice, setDevice); + return builder(devices, currentDevice, (devices) { + setDevice(devices, isCm, isVoiceCall); + }); }, ); } diff --git a/flutter/lib/desktop/pages/desktop_setting_page.dart b/flutter/lib/desktop/pages/desktop_setting_page.dart index 8ff417585..505a98f17 100644 --- a/flutter/lib/desktop/pages/desktop_setting_page.dart +++ b/flutter/lib/desktop/pages/desktop_setting_page.dart @@ -500,7 +500,7 @@ class _GeneralState extends State<_General> { return const Offstage(); } - return AudioInput(builder: (devices, currentDevice, setDevice) { + builder(devices, currentDevice, setDevice) { return _Card(title: 'Audio Input Device', children: [ ...devices.map((device) => _Radio(context, value: device, @@ -511,7 +511,9 @@ class _GeneralState extends State<_General> { setState(() {}); })) ]); - }); + } + + return AudioInput(builder: builder, isCm: false, isVoiceCall: false); } Widget record(BuildContext context) { diff --git a/flutter/lib/desktop/pages/server_page.dart b/flutter/lib/desktop/pages/server_page.dart index 1e63c71bd..1cac2706e 100644 --- a/flutter/lib/desktop/pages/server_page.dart +++ b/flutter/lib/desktop/pages/server_page.dart @@ -732,7 +732,7 @@ class _CmControlPanel extends StatelessWidget { child: buildButton(context, color: MyTheme.accent, onClick: null, onTapDown: (details) async { - final devicesInfo = await AudioInput.getDevicesInfo(); + final devicesInfo = await AudioInput.getDevicesInfo(true, true); List devices = devicesInfo['devices'] as List; if (devices.isEmpty) { msgBox( @@ -758,13 +758,13 @@ class _CmControlPanel extends StatelessWidget { value: d, height: 18, padding: EdgeInsets.zero, - onTap: () => AudioInput.setDevice(d), + onTap: () => AudioInput.setDevice(d, true, true), child: IgnorePointer( child: RadioMenuButton( value: d, groupValue: currentDevice, onChanged: (v) { - if (v != null) AudioInput.setDevice(v); + if (v != null) AudioInput.setDevice(v, true, true); }, child: Container( child: Text( diff --git a/flutter/lib/desktop/widgets/remote_toolbar.dart b/flutter/lib/desktop/widgets/remote_toolbar.dart index 5b649049c..d90e24f53 100644 --- a/flutter/lib/desktop/widgets/remote_toolbar.dart +++ b/flutter/lib/desktop/widgets/remote_toolbar.dart @@ -1984,28 +1984,31 @@ class _VoiceCallMenu extends StatelessWidget { @override Widget build(BuildContext context) { menuChildrenGetter() { - final audioInput = - AudioInput(builder: (devices, currentDevice, setDevice) { - return Column( - children: devices - .map((d) => RdoMenuButton( - child: Container( - child: Text( - d, - overflow: TextOverflow.ellipsis, + final audioInput = AudioInput( + builder: (devices, currentDevice, setDevice) { + return Column( + children: devices + .map((d) => RdoMenuButton( + child: Container( + child: Text( + d, + overflow: TextOverflow.ellipsis, + ), + constraints: BoxConstraints(maxWidth: 250), ), - constraints: BoxConstraints(maxWidth: 250), - ), - value: d, - groupValue: currentDevice, - onChanged: (v) { - if (v != null) setDevice(v); - }, - ffi: ffi, - )) - .toList(), - ); - }); + value: d, + groupValue: currentDevice, + onChanged: (v) { + if (v != null) setDevice(v); + }, + ffi: ffi, + )) + .toList(), + ); + }, + isCm: false, + isVoiceCall: true, + ); return [ audioInput, Divider(), diff --git a/src/client/io_loop.rs b/src/client/io_loop.rs index 19074bbd1..15a779109 100644 --- a/src/client/io_loop.rs +++ b/src/client/io_loop.rs @@ -42,7 +42,7 @@ use crate::client::{ }; #[cfg(not(any(target_os = "android", target_os = "ios")))] use crate::clipboard::{update_clipboard, CLIPBOARD_INTERVAL}; -use crate::common::{get_default_sound_input, set_sound_input}; +use crate::common::get_default_sound_input; use crate::ui_session_interface::{InvokeUiSession, Session}; #[cfg(not(any(target_os = "ios")))] use crate::{audio_service, ConnInner, CLIENT_SERVER}; @@ -387,11 +387,12 @@ impl Remote { if self.handler.is_file_transfer() || self.handler.is_port_forward() { return None; } - // Switch to default input device - let default_sound_device = get_default_sound_input(); - if let Some(device) = default_sound_device { - set_sound_input(device); - } + // NOTE: + // The client server and --server both use the same sound input device. + // It's better to distinguish the server side and client side. + // But it' not necessary for now, because it's not a common case. + // And it is immediately known when the input device is changed. + crate::audio_service::set_voice_call_input_device(get_default_sound_input(), false); // iOS does not have this server. #[cfg(not(any(target_os = "ios")))] { @@ -421,6 +422,7 @@ impl Remote { client_conn_inner, false, ); + crate::audio_service::set_voice_call_input_device(None, true); break; } _ => {} diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index 5371b225e..3eccfc84d 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -1319,6 +1319,25 @@ pub fn cm_close_voice_call(id: i32) { crate::ui_cm_interface::close_voice_call(id); } +pub fn set_voice_call_input_device(is_cm: bool, device: String) { + if is_cm { + let _ = crate::ipc::set_config("voice-call-input", device); + } else { + crate::audio_service::set_voice_call_input_device(Some(device), true); + } +} + +pub fn get_voice_call_input_device(is_cm: bool) -> String { + if is_cm { + match crate::ipc::get_config("voice-call-input") { + Ok(Some(device)) => device, + _ => "".to_owned(), + } + } else { + crate::audio_service::get_voice_call_input_device().unwrap_or_default() + } +} + pub fn main_get_last_remote_id() -> String { LocalConfig::get_remote_id() } diff --git a/src/ipc.rs b/src/ipc.rs index 20e10e1b2..489db38e7 100644 --- a/src/ipc.rs +++ b/src/ipc.rs @@ -308,7 +308,7 @@ pub async fn new_listener(postfix: &str) -> ResultType { } } -pub struct CheckIfRestart(String, Vec, String); +pub struct CheckIfRestart(String, Vec, String, String); impl CheckIfRestart { pub fn new() -> CheckIfRestart { @@ -316,6 +316,7 @@ impl CheckIfRestart { Config::get_option("stop-service"), Config::get_rendezvous_servers(), Config::get_option("audio-input"), + Config::get_option("voice-call-input"), ) } } @@ -329,6 +330,12 @@ impl Drop for CheckIfRestart { if self.2 != Config::get_option("audio-input") { crate::audio_service::restart(); } + if self.3 != Config::get_option("voice-call-input") { + crate::audio_service::set_voice_call_input_device( + Some(Config::get_option("voice-call-input")), + true, + ) + } } } @@ -457,6 +464,8 @@ async fn handle(data: Data, stream: &mut Connection) { } else { None }; + } else if name == "voice-call-input" { + value = crate::audio_service::get_voice_call_input_device(); } else { value = None; } @@ -472,6 +481,8 @@ async fn handle(data: Data, stream: &mut Connection) { Config::set_permanent_password(&value); } else if name == "salt" { Config::set_salt(&value); + } else if name == "voice-call-input" { + crate::audio_service::set_voice_call_input_device(Some(value), true); } else { return; } @@ -488,12 +499,7 @@ async fn handle(data: Data, stream: &mut Connection) { if let Some(v) = value.get("privacy-mode-impl-key") { crate::privacy_mode::switch(v); } - let pre_opts = Config::get_options(); - let new_audio_input = pre_opts.get("audio-input"); Config::set_options(value); - if new_audio_input != pre_opts.get("audio-input") { - crate::audio_service::restart(); - } allow_err!(stream.send(&Data::Options(None)).await); } }, diff --git a/src/server/audio_service.rs b/src/server/audio_service.rs index cfe7b457b..504c09279 100644 --- a/src/server/audio_service.rs +++ b/src/server/audio_service.rs @@ -22,6 +22,10 @@ pub const NAME: &'static str = "audio"; pub const AUDIO_DATA_SIZE_U8: usize = 960 * 4; // 10ms in 48000 stereo static RESTARTING: AtomicBool = AtomicBool::new(false); +lazy_static::lazy_static! { + static ref VOICE_CALL_INPUT_DEVICE: Arc::>> = Default::default(); +} + #[cfg(not(any(target_os = "linux", target_os = "android")))] pub fn new() -> GenericService { let svc = EmptyExtraFieldService::new(NAME.to_owned(), true); @@ -36,6 +40,33 @@ pub fn new() -> GenericService { svc.sp } +#[inline] +pub fn get_voice_call_input_device() -> Option { + VOICE_CALL_INPUT_DEVICE.lock().unwrap().clone() +} + +#[inline] +pub fn set_voice_call_input_device(device: Option, set_if_present: bool) { + if !set_if_present && VOICE_CALL_INPUT_DEVICE.lock().unwrap().is_some() { + return; + } + + if *VOICE_CALL_INPUT_DEVICE.lock().unwrap() == device { + return; + } + *VOICE_CALL_INPUT_DEVICE.lock().unwrap() = device; + restart(); +} + +#[inline] +fn get_audio_input() -> String { + VOICE_CALL_INPUT_DEVICE + .lock() + .unwrap() + .clone() + .unwrap_or(Config::get_option("audio-input")) +} + pub fn restart() { log::info!("restart the audio service, freezing now..."); if RESTARTING.load(Ordering::SeqCst) { @@ -62,7 +93,7 @@ mod pa_impl { stream .send(&crate::ipc::Data::Config(( "audio-input".to_owned(), - Some(Config::get_option("audio-input")) + Some(super::get_audio_input()) ))) .await ); @@ -132,6 +163,7 @@ mod cpal_impl { } fn run_restart(sp: EmptyExtraFieldService, state: &mut State) -> ResultType<()> { + println!("REMOVE ME ========================= run_restart"); state.reset(); sp.snapshot(|_sps: ServiceSwap<_>| Ok(()))?; match &state.stream { @@ -198,7 +230,8 @@ mod cpal_impl { #[cfg(windows)] fn get_device() -> ResultType<(Device, SupportedStreamConfig)> { - let audio_input = Config::get_option("audio-input"); + let audio_input = super::get_audio_input(); + println!("REMOVE ME =============================== use audio input: {}", &audio_input); if !audio_input.is_empty() { return get_audio_input(&audio_input); } @@ -219,7 +252,7 @@ mod cpal_impl { #[cfg(not(windows))] fn get_device() -> ResultType<(Device, SupportedStreamConfig)> { - let audio_input = Config::get_option("audio-input"); + let audio_input = super::get_audio_input(); get_audio_input(&audio_input) } diff --git a/src/server/connection.rs b/src/server/connection.rs index fd3dd2a03..4a383fe35 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -17,7 +17,6 @@ use crate::{ client::{ new_voice_call_request, new_voice_call_response, start_audio_thread, MediaData, MediaSender, }, - common::{get_default_sound_input, set_sound_input}, display_service, ipc, privacy_mode, video_service, VERSION, }; #[cfg(any(target_os = "android", target_os = "ios"))] @@ -218,7 +217,6 @@ pub struct Connection { portable: PortableState, from_switch: bool, voice_call_request_timestamp: Option, - audio_input_device_before_voice_call: Option, options_in_login: Option, #[cfg(not(any(target_os = "ios")))] pressed_modifiers: HashSet, @@ -367,7 +365,6 @@ impl Connection { from_switch: false, audio_sender: None, voice_call_request_timestamp: None, - audio_input_device_before_voice_call: None, options_in_login: None, #[cfg(not(any(target_os = "ios")))] pressed_modifiers: Default::default(), @@ -2061,12 +2058,13 @@ impl Connection { cb.content.into() }; if let Ok(content) = String::from_utf8(content) { - let data = HashMap::from([ - ("name", "clipboard"), - ("content", &content), - ]); + let data = + HashMap::from([("name", "clipboard"), ("content", &content)]); if let Ok(data) = serde_json::to_string(&data) { - let _ = crate::flutter::push_global_event(crate::flutter::APP_TYPE_MAIN, data); + let _ = crate::flutter::push_global_event( + crate::flutter::APP_TYPE_MAIN, + data, + ); } } } @@ -2720,15 +2718,10 @@ impl Connection { if let Some(ts) = self.voice_call_request_timestamp.take() { let msg = new_voice_call_response(ts.get(), accepted); if accepted { - // Backup the default input device. - let audio_input_device = Config::get_option("audio-input"); - log::debug!("Backup the sound input device {}", audio_input_device); - self.audio_input_device_before_voice_call = Some(audio_input_device); - // Switch to default input device - let default_sound_device = get_default_sound_input(); - if let Some(device) = default_sound_device { - set_sound_input(device); - } + crate::audio_service::set_voice_call_input_device( + crate::get_default_sound_input(), + false, + ); self.send_to_cm(Data::StartVoiceCall); } else { self.send_to_cm(Data::CloseVoiceCall("".to_owned())); @@ -2740,12 +2733,7 @@ impl Connection { } pub async fn close_voice_call(&mut self) { - // Restore to the prior audio device. - if let Some(sound_input) = - std::mem::replace(&mut self.audio_input_device_before_voice_call, None) - { - set_sound_input(sound_input); - } + crate::audio_service::set_voice_call_input_device(None, true); // Notify the connection manager that the voice call has been closed. self.send_to_cm(Data::CloseVoiceCall("".to_owned())); } @@ -3035,6 +3023,16 @@ impl Connection { return; } self.closed = true; + // If voice A,B -> C, and A,B has voice call + // B disconnects, C will reset the voice call input. + // + // It may be acceptable, because it's not a common case, + // and it's immediately known when the input device changes. + // C can change the input device manually in cm interface. + // + // We can add a (Vec, input device) to avoid this. + // But it's not necessary now and we have to consider two audio services(client, server). + crate::audio_service::set_voice_call_input_device(None, true); log::info!("#{} Connection closed: {}", self.inner.id(), reason); if lock && self.lock_after_session_end && self.keyboard { #[cfg(not(any(target_os = "android", target_os = "ios")))]