feat: macos, audio, loopback (#10025)

Signed-off-by: fufesou <linlong1266@gmail.com>

.github/workflows/flutter-build.yml

@@ -20,6 +20,7 @@ on:
 env:
   SCITER_RUST_VERSION: "1.75" # https://github.com/rustdesk/rustdesk/discussions/7503, also 1.78 has ABI change which causes our sciter version not working, https://blog.rust-lang.org/2024/03/30/i128-layout-update.html
   RUST_VERSION: "1.75" # sciter failed on m1 with 1.78 because of https://blog.rust-lang.org/2024/03/30/i128-layout-update.html
+  MAC_RUST_VERSION: "1.81" # 1.81 is requred for macos, because of https://github.com/yury/cidre requires 1.81
   CARGO_NDK_VERSION: "3.1.2"
   SCITER_ARMV7_CMAKE_VERSION: "3.29.7"
   SCITER_NASM_DEBVERSION: "2.14-1"
@@ -641,7 +642,7 @@ jobs:
               target: aarch64-apple-darwin,
               os: macos-latest,
               # extra-build-args: "--disable-flutter-texture-render", # disable this for mac, because we see a lot of users reporting flickering both on arm and x64, and we can not confirm if texture rendering has better performance if htere is no vram, https://github.com/rustdesk/rustdesk/issues/6296
-              extra-build-args: "",
+              extra-build-args: "--screencapturekit",
               arch: aarch64,
             }
     steps:
@@ -720,7 +721,7 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@v1
         with:
-          toolchain: ${{ env.RUST_VERSION }}
+          toolchain: ${{ env.MAC_RUST_VERSION }}
           targets: ${{ matrix.job.target }}
           components: "rustfmt"
 
@@ -767,6 +768,13 @@ jobs:
 
       - name: Build rustdesk
         run: |
+          if [ "${{ matrix.job.target }}" = "aarch64-apple-darwin" ]; then
+            MIN_MACOS_VERSION="12.3"
+            sed -i -e "s/MACOSX_DEPLOYMENT_TARGET\=[0-9]*.[0-9]*/MACOSX_DEPLOYMENT_TARGET=${MIN_MACOS_VERSION}/" build.py
+            sed -i -e "s/platform :osx, '.*'/platform :osx, '${MIN_MACOS_VERSION}'/" flutter/macos/Podfile
+            sed -i -e "s/osx_minimum_system_version = \"[0-9]*.[0-9]*\"/osx_minimum_system_version = \"${MIN_MACOS_VERSION}\"/" Cargo.toml
+            sed -i -e "s/MACOSX_DEPLOYMENT_TARGET = [0-9]*.[0-9]*;/MACOSX_DEPLOYMENT_TARGET = ${MIN_MACOS_VERSION};/" flutter/macos/Runner.xcodeproj/project.pbxproj
+          fi
           ./build.py --flutter --hwcodec ${{ matrix.job.extra-build-args }}
 
       - name: create unsigned dmg

Cargo.lock

@@ -893,6 +893,20 @@ dependencies = [
  "regex",
 ]
 
+[[package]]
+name = "cidre"
+version = "0.4.0"
+source = "git+https://github.com/yury/cidre.git?rev=f05c428#f05c4288f9870c9fab53272ddafd6ec01c7b2dbf"
+dependencies = [
+ "cidre-macros",
+ "parking_lot",
+]
+
+[[package]]
+name = "cidre-macros"
+version = "0.1.0"
+source = "git+https://github.com/yury/cidre.git?rev=f05c428#f05c4288f9870c9fab53272ddafd6ec01c7b2dbf"
+
 [[package]]
 name = "cipher"
 version = "0.4.4"
@@ -1276,10 +1290,10 @@ dependencies = [
 [[package]]
 name = "cpal"
 version = "0.15.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779"
+source = "git+https://github.com/rustdesk-org/cpal?branch=osx-screencapturekit#4d318ff778063ce14669fd4bd67a1673653fc6e5"
 dependencies = [
  "alsa",
+ "cidre",
  "core-foundation-sys 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "coreaudio-rs",
  "dasp_sample",
@@ -3505,7 +3519,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d"
 dependencies = [
  "cfg-if 1.0.0",
- "windows-targets 0.52.5",
+ "windows-targets 0.48.5",
 ]
 
 [[package]]
@@ -4149,7 +4163,7 @@ version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b"
 dependencies = [
- "proc-macro-crate 2.0.2",
+ "proc-macro-crate 1.3.1",
  "proc-macro2 1.0.86",
  "quote 1.0.36",
  "syn 2.0.68",

Cargo.toml

@@ -36,6 +36,7 @@ unix-file-copy-paste = [
     "dep:once_cell",
     "clipboard/unix-file-copy-paste",
 ]
+screencapturekit = ["cpal/screencapturekit"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -77,7 +78,7 @@ fon = "0.6"
 zip = "0.6"
 shutdown_hooks = "0.1"
 totp-rs = { version = "5.4", default-features = false, features = ["gen_secret", "otpauth"] }
-cpal = "0.15"
+cpal = { git = "https://github.com/rustdesk-org/cpal", branch = "osx-screencapturekit" }
 ringbuf = "0.3"
 
 [target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies]

build.py

@@ -143,6 +143,12 @@ def make_parser():
         "--package",
         type=str
     )
+    if osx:
+        parser.add_argument(
+            '--screencapturekit',
+            action='store_true',
+            help='Enable feature screencapturekit'
+        )
     return parser
 
 
@@ -274,6 +280,9 @@ def get_features(args):
         features.append('flutter')
     if args.unix_file_copy_paste:
         features.append('unix-file-copy-paste')
+    if osx:
+        if args.screencapturekit:
+            features.append('screencapturekit')
     print("features:", features)
     return features
 

flutter/lib/common/widgets/audio_input.dart

@@ -2,7 +2,7 @@ import 'package:flutter/material.dart';
 import 'package:flutter_hbb/common.dart';
 import 'package:flutter_hbb/models/platform_model.dart';
 
-const _kWindowsSystemSound = 'System Sound';
+const _kSystemSound = 'System Sound';
 
 typedef AudioINputSetDevice = void Function(String device);
 typedef AudioInputBuilder = Widget Function(
@@ -21,7 +21,7 @@ class AudioInput extends StatelessWidget {
       : super(key: key);
 
   static String getDefault() {
-    if (isWindows) return translate('System Sound');
+    if (bind.mainAudioSupportLoopback()) return translate(_kSystemSound);
     return '';
   }
 
@@ -55,8 +55,8 @@ class AudioInput extends StatelessWidget {
   static Future<Map<String, Object>> getDevicesInfo(
       bool isCm, bool isVoiceCall) async {
     List<String> devices = (await bind.mainGetSoundInputs()).toList();
-    if (isWindows) {
-      devices.insert(0, translate(_kWindowsSystemSound));
+    if (bind.mainAudioSupportLoopback()) {
+      devices.insert(0, translate(_kSystemSound));
     }
     String current = await getValue(isCm, isVoiceCall);
     return {'devices': devices, 'current': current};

src/flutter_ffi.rs

@@ -774,13 +774,6 @@ pub fn main_get_sound_inputs() -> Vec<String> {
     vec![String::from("")]
 }
 
-pub fn main_get_default_sound_input() -> Option<String> {
-    #[cfg(not(any(target_os = "android", target_os = "ios")))]
-    return get_default_sound_input();
-    #[cfg(any(target_os = "android", target_os = "ios"))]
-    None
-}
-
 pub fn main_get_login_device_info() -> SyncReturn<String> {
     SyncReturn(get_login_device_info_json())
 }
@@ -2317,6 +2310,16 @@ pub fn session_request_new_display_init_msgs(session_id: SessionID, display: usi
     }
 }
 
+pub fn main_audio_support_loopback() -> SyncReturn<bool> {
+    #[cfg(target_os = "windows")]
+    let is_surpport = true;
+    #[cfg(feature = "screencapturekit")]
+    let is_surpport = crate::audio_service::is_screen_capture_kit_available();
+    #[cfg(not(any(target_os = "windows", feature = "screencapturekit")))]
+    let is_surpport = false;
+    SyncReturn(is_surpport)
+}
+
 #[cfg(target_os = "android")]
 pub mod server_side {
     use hbb_common::{config, log};

src/server/audio_service.rs

@@ -156,6 +156,14 @@ mod pa_impl {
     }
 }
 
+#[inline]
+#[cfg(feature = "screencapturekit")]
+pub fn is_screen_capture_kit_available() -> bool {
+    cpal::available_hosts()
+        .iter()
+        .any(|host| *host == cpal::HostId::ScreenCaptureKit)
+}
+
 #[cfg(not(any(target_os = "linux", target_os = "android")))]
 mod cpal_impl {
     use self::service::{Reset, ServiceSwap};
@@ -170,6 +178,11 @@ mod cpal_impl {
         static ref INPUT_BUFFER: Arc<Mutex<std::collections::VecDeque<f32>>> = Default::default();
     }
 
+    #[cfg(feature = "screencapturekit")]
+    lazy_static::lazy_static! {
+        static ref HOST_SCREEN_CAPTURE_KIT: Result<Host, cpal::HostUnavailable> = cpal::host_from_id(cpal::HostId::ScreenCaptureKit);
+    }
+
     #[derive(Default)]
     pub struct State {
         stream: Option<(Box<dyn StreamTrait>, Arc<Message>)>,
@@ -246,6 +259,27 @@ mod cpal_impl {
         send_f32(&data, encoder, sp);
     }
 
+    #[cfg(feature = "screencapturekit")]
+    fn get_device() -> ResultType<(Device, SupportedStreamConfig)> {
+        let audio_input = super::get_audio_input();
+        if !audio_input.is_empty() {
+            return get_audio_input(&audio_input);
+        }
+        if !is_screen_capture_kit_available() {
+            return get_audio_input("");
+        }
+        let device = HOST_SCREEN_CAPTURE_KIT
+            .as_ref()?
+            .default_input_device()
+            .with_context(|| "Failed to get default input device for loopback")?;
+        let format = device
+            .default_input_config()
+            .map_err(|e| anyhow!(e))
+            .with_context(|| "Failed to get input output format")?;
+        log::info!("Default input format: {:?}", format);
+        Ok((device, format))
+    }
+
     #[cfg(windows)]
     fn get_device() -> ResultType<(Device, SupportedStreamConfig)> {
         let audio_input = super::get_audio_input();
@@ -267,7 +301,7 @@ mod cpal_impl {
         Ok((device, format))
     }
 
-    #[cfg(not(windows))]
+    #[cfg(not(any(windows, feature = "screencapturekit")))]
     fn get_device() -> ResultType<(Device, SupportedStreamConfig)> {
         let audio_input = super::get_audio_input();
         get_audio_input(&audio_input)
@@ -275,7 +309,20 @@ mod cpal_impl {
 
     fn get_audio_input(audio_input: &str) -> ResultType<(Device, SupportedStreamConfig)> {
         let mut device = None;
-        if !audio_input.is_empty() {
+        #[cfg(feature = "screencapturekit")]
+        if !audio_input.is_empty() && is_screen_capture_kit_available() {
+            for d in HOST_SCREEN_CAPTURE_KIT
+                .as_ref()?
+                .devices()
+                .with_context(|| "Failed to get audio devices")?
+            {
+                if d.name().unwrap_or("".to_owned()) == audio_input {
+                    device = Some(d);
+                    break;
+                }
+            }
+        }
+        if device.is_none() && !audio_input.is_empty() {
             for d in HOST
                 .devices()
                 .with_context(|| "Failed to get audio devices")?

src/ui_interface.rs

@@ -321,6 +321,8 @@ pub fn get_sound_inputs() -> Vec<String> {
         fn get_sound_inputs_() -> Vec<String> {
             let mut out = Vec::new();
             use cpal::traits::{DeviceTrait, HostTrait};
+            // Do not use `cpal::host_from_id(cpal::HostId::ScreenCaptureKit)` for feature = "screencapturekit"
+            // Because we explicitly handle the "System Sound" device.
             let host = cpal::default_host();
             if let Ok(devices) = host.devices() {
                 for device in devices {