audio rechannel

Signed-off-by: 21pages <pages21@163.com>

Cargo.lock

@@ -2083,6 +2083,15 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "fon"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad46a0e6c9bc688823a742aa969b5c08fdc56c2a436ee00d5c6fbcb5982c55c4"
+dependencies = [
+ "libm",
+]
+
 [[package]]
 name = "foreign-types"
 version = "0.3.2"
@@ -3293,6 +3302,12 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "libm"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
+
 [[package]]
 name = "libpulse-binding"
 version = "2.27.1"
@@ -5124,6 +5139,7 @@ dependencies = [
  "evdev",
  "flutter_rust_bridge",
  "flutter_rust_bridge_codegen",
+ "fon",
  "fruitbasket",
  "hbb_common",
  "hex",

Cargo.toml

@@ -71,6 +71,7 @@ reqwest = { version = "0.11", features = ["blocking", "json", "rustls-tls"], def
 chrono = "0.4"
 cidr-utils = "0.5"
 libloading = "0.7"
+fon = "0.6"
 
 [target.'cfg(not(any(target_os = "android", target_os = "linux")))'.dependencies]
 cpal = "0.15"

src/client.rs

@@ -753,6 +753,7 @@ pub struct AudioHandler {
     #[cfg(not(any(target_os = "android", target_os = "linux")))]
     audio_stream: Option<Box<dyn StreamTrait>>,
     channels: u16,
+    device_channel: u16,
     #[cfg(not(any(target_os = "android", target_os = "linux")))]
     ready: Arc<std::sync::Mutex<bool>>,
 }
@@ -825,8 +826,8 @@ impl AudioHandler {
         let sample_format = config.sample_format();
         log::info!("Default output format: {:?}", config);
         log::info!("Remote input format: {:?}", format0);
-        let mut config: StreamConfig = config.into();
+        let config: StreamConfig = config.into();
-        config.channels = format0.channels as _;
+        self.device_channel = config.channels;
         match sample_format {
             cpal::SampleFormat::I8 => self.build_output_stream::<i8>(&config, &device)?,
             cpal::SampleFormat::I16 => self.build_output_stream::<i16>(&config, &device)?,
@@ -884,20 +885,25 @@ impl AudioHandler {
                     let sample_rate0 = self.sample_rate.0;
                     let sample_rate = self.sample_rate.1;
                     let audio_buffer = self.audio_buffer.0.clone();
+                    let mut buffer = buffer[0..n].to_owned();
                     if sample_rate != sample_rate0 {
-                        let buffer = crate::resample_channels(
+                        buffer = crate::audio_resample(
                             &buffer[0..n],
                             sample_rate0,
                             sample_rate,
                             channels,
                         );
-                        audio_buffer.lock().unwrap().push_slice_overwrite(&buffer);
-                    } else {
-                        audio_buffer
-                            .lock()
-                            .unwrap()
-                            .push_slice_overwrite(&buffer[0..n]);
                     }
+                    if self.channels != self.device_channel {
+                        buffer = crate::audio_rechannel(
+                            buffer,
+                            sample_rate,
+                            sample_rate,
+                            self.channels,
+                            self.device_channel,
+                        );
+                    }
+                    audio_buffer.lock().unwrap().push_slice_overwrite(&buffer);
                 }
                 #[cfg(target_os = "android")]
                 {

src/common.rs

@@ -273,7 +273,7 @@ pub fn resample_channels(
 }
 
 #[cfg(feature = "use_dasp")]
-pub fn resample_channels(
+pub fn audio_resample(
     data: &[f32],
     sample_rate0: u32,
     sample_rate: u32,
@@ -309,7 +309,7 @@ pub fn resample_channels(
 }
 
 #[cfg(feature = "use_samplerate")]
-pub fn resample_channels(
+pub fn audio_resample(
     data: &[f32],
     sample_rate0: u32,
     sample_rate: u32,
@@ -326,6 +326,158 @@ pub fn resample_channels(
     .unwrap_or_default()
 }
 
+pub fn audio_rechannel(
+    input: Vec<f32>,
+    in_hz: u32,
+    out_hz: u32,
+    in_chan: u16,
+    output_chan: u16,
+) -> Vec<f32> {
+    if in_chan == output_chan {
+        return input;
+    }
+    let mut input = input;
+    input.truncate(input.len() / in_chan as usize * in_chan as usize);
+    match (in_chan, output_chan) {
+        (1, 2) => audio_rechannel_1_2(&input, in_hz, out_hz),
+        (1, 3) => audio_rechannel_1_3(&input, in_hz, out_hz),
+        (1, 4) => audio_rechannel_1_4(&input, in_hz, out_hz),
+        (1, 5) => audio_rechannel_1_5(&input, in_hz, out_hz),
+        (1, 6) => audio_rechannel_1_6(&input, in_hz, out_hz),
+        (1, 7) => audio_rechannel_1_7(&input, in_hz, out_hz),
+        (1, 8) => audio_rechannel_1_8(&input, in_hz, out_hz),
+        (2, 1) => audio_rechannel_2_1(&input, in_hz, out_hz),
+        (2, 3) => audio_rechannel_2_3(&input, in_hz, out_hz),
+        (2, 4) => audio_rechannel_2_4(&input, in_hz, out_hz),
+        (2, 5) => audio_rechannel_2_5(&input, in_hz, out_hz),
+        (2, 6) => audio_rechannel_2_6(&input, in_hz, out_hz),
+        (2, 7) => audio_rechannel_2_7(&input, in_hz, out_hz),
+        (2, 8) => audio_rechannel_2_8(&input, in_hz, out_hz),
+        (3, 1) => audio_rechannel_3_1(&input, in_hz, out_hz),
+        (3, 2) => audio_rechannel_3_2(&input, in_hz, out_hz),
+        (3, 4) => audio_rechannel_3_4(&input, in_hz, out_hz),
+        (3, 5) => audio_rechannel_3_5(&input, in_hz, out_hz),
+        (3, 6) => audio_rechannel_3_6(&input, in_hz, out_hz),
+        (3, 7) => audio_rechannel_3_7(&input, in_hz, out_hz),
+        (3, 8) => audio_rechannel_3_8(&input, in_hz, out_hz),
+        (4, 1) => audio_rechannel_4_1(&input, in_hz, out_hz),
+        (4, 2) => audio_rechannel_4_2(&input, in_hz, out_hz),
+        (4, 3) => audio_rechannel_4_3(&input, in_hz, out_hz),
+        (4, 5) => audio_rechannel_4_5(&input, in_hz, out_hz),
+        (4, 6) => audio_rechannel_4_6(&input, in_hz, out_hz),
+        (4, 7) => audio_rechannel_4_7(&input, in_hz, out_hz),
+        (4, 8) => audio_rechannel_4_8(&input, in_hz, out_hz),
+        (5, 1) => audio_rechannel_5_1(&input, in_hz, out_hz),
+        (5, 2) => audio_rechannel_5_2(&input, in_hz, out_hz),
+        (5, 3) => audio_rechannel_5_3(&input, in_hz, out_hz),
+        (5, 4) => audio_rechannel_5_4(&input, in_hz, out_hz),
+        (5, 6) => audio_rechannel_5_6(&input, in_hz, out_hz),
+        (5, 7) => audio_rechannel_5_7(&input, in_hz, out_hz),
+        (5, 8) => audio_rechannel_5_8(&input, in_hz, out_hz),
+        (6, 1) => audio_rechannel_6_1(&input, in_hz, out_hz),
+        (6, 2) => audio_rechannel_6_2(&input, in_hz, out_hz),
+        (6, 3) => audio_rechannel_6_3(&input, in_hz, out_hz),
+        (6, 4) => audio_rechannel_6_4(&input, in_hz, out_hz),
+        (6, 5) => audio_rechannel_6_5(&input, in_hz, out_hz),
+        (6, 7) => audio_rechannel_6_7(&input, in_hz, out_hz),
+        (6, 8) => audio_rechannel_6_8(&input, in_hz, out_hz),
+        (7, 1) => audio_rechannel_7_1(&input, in_hz, out_hz),
+        (7, 2) => audio_rechannel_7_2(&input, in_hz, out_hz),
+        (7, 3) => audio_rechannel_7_3(&input, in_hz, out_hz),
+        (7, 4) => audio_rechannel_7_4(&input, in_hz, out_hz),
+        (7, 5) => audio_rechannel_7_5(&input, in_hz, out_hz),
+        (7, 6) => audio_rechannel_7_6(&input, in_hz, out_hz),
+        (7, 8) => audio_rechannel_7_8(&input, in_hz, out_hz),
+        (8, 1) => audio_rechannel_8_1(&input, in_hz, out_hz),
+        (8, 2) => audio_rechannel_8_2(&input, in_hz, out_hz),
+        (8, 3) => audio_rechannel_8_3(&input, in_hz, out_hz),
+        (8, 4) => audio_rechannel_8_4(&input, in_hz, out_hz),
+        (8, 5) => audio_rechannel_8_5(&input, in_hz, out_hz),
+        (8, 6) => audio_rechannel_8_6(&input, in_hz, out_hz),
+        (8, 7) => audio_rechannel_8_7(&input, in_hz, out_hz),
+        _ => input,
+    }
+}
+
+macro_rules! audio_rechannel {
+    ($name:ident, $in_channels:expr, $out_channels:expr) => {
+        fn $name(input: &[f32], in_hz: u32, out_hz: u32) -> Vec<f32> {
+            use fon::{chan::Ch32, Audio, Frame};
+            let mut in_audio =
+                Audio::<Ch32, $in_channels>::with_silence(in_hz, input.len() / $in_channels);
+            for (x, y) in input.chunks_exact($in_channels).zip(in_audio.iter_mut()) {
+                let mut f = Frame::<Ch32, $in_channels>::default();
+                let mut i = 0;
+                for c in f.channels_mut() {
+                    *c = x[i].into();
+                    i += 1;
+                }
+                *y = f;
+            }
+            Audio::<Ch32, $out_channels>::with_audio(out_hz, &in_audio)
+                .as_f32_slice()
+                .to_owned()
+        }
+    };
+}
+
+audio_rechannel!(audio_rechannel_1_2, 1, 2);
+audio_rechannel!(audio_rechannel_1_3, 1, 3);
+audio_rechannel!(audio_rechannel_1_4, 1, 4);
+audio_rechannel!(audio_rechannel_1_5, 1, 5);
+audio_rechannel!(audio_rechannel_1_6, 1, 6);
+audio_rechannel!(audio_rechannel_1_7, 1, 7);
+audio_rechannel!(audio_rechannel_1_8, 1, 8);
+audio_rechannel!(audio_rechannel_2_1, 2, 1);
+audio_rechannel!(audio_rechannel_2_3, 2, 3);
+audio_rechannel!(audio_rechannel_2_4, 2, 4);
+audio_rechannel!(audio_rechannel_2_5, 2, 5);
+audio_rechannel!(audio_rechannel_2_6, 2, 6);
+audio_rechannel!(audio_rechannel_2_7, 2, 7);
+audio_rechannel!(audio_rechannel_2_8, 2, 8);
+audio_rechannel!(audio_rechannel_3_1, 3, 1);
+audio_rechannel!(audio_rechannel_3_2, 3, 2);
+audio_rechannel!(audio_rechannel_3_4, 3, 4);
+audio_rechannel!(audio_rechannel_3_5, 3, 5);
+audio_rechannel!(audio_rechannel_3_6, 3, 6);
+audio_rechannel!(audio_rechannel_3_7, 3, 7);
+audio_rechannel!(audio_rechannel_3_8, 3, 8);
+audio_rechannel!(audio_rechannel_4_1, 4, 1);
+audio_rechannel!(audio_rechannel_4_2, 4, 2);
+audio_rechannel!(audio_rechannel_4_3, 4, 3);
+audio_rechannel!(audio_rechannel_4_5, 4, 5);
+audio_rechannel!(audio_rechannel_4_6, 4, 6);
+audio_rechannel!(audio_rechannel_4_7, 4, 7);
+audio_rechannel!(audio_rechannel_4_8, 4, 8);
+audio_rechannel!(audio_rechannel_5_1, 5, 1);
+audio_rechannel!(audio_rechannel_5_2, 5, 2);
+audio_rechannel!(audio_rechannel_5_3, 5, 3);
+audio_rechannel!(audio_rechannel_5_4, 5, 4);
+audio_rechannel!(audio_rechannel_5_6, 5, 6);
+audio_rechannel!(audio_rechannel_5_7, 5, 7);
+audio_rechannel!(audio_rechannel_5_8, 5, 8);
+audio_rechannel!(audio_rechannel_6_1, 6, 1);
+audio_rechannel!(audio_rechannel_6_2, 6, 2);
+audio_rechannel!(audio_rechannel_6_3, 6, 3);
+audio_rechannel!(audio_rechannel_6_4, 6, 4);
+audio_rechannel!(audio_rechannel_6_5, 6, 5);
+audio_rechannel!(audio_rechannel_6_7, 6, 7);
+audio_rechannel!(audio_rechannel_6_8, 6, 8);
+audio_rechannel!(audio_rechannel_7_1, 7, 1);
+audio_rechannel!(audio_rechannel_7_2, 7, 2);
+audio_rechannel!(audio_rechannel_7_3, 7, 3);
+audio_rechannel!(audio_rechannel_7_4, 7, 4);
+audio_rechannel!(audio_rechannel_7_5, 7, 5);
+audio_rechannel!(audio_rechannel_7_6, 7, 6);
+audio_rechannel!(audio_rechannel_7_8, 7, 8);
+audio_rechannel!(audio_rechannel_8_1, 8, 1);
+audio_rechannel!(audio_rechannel_8_2, 8, 2);
+audio_rechannel!(audio_rechannel_8_3, 8, 3);
+audio_rechannel!(audio_rechannel_8_4, 8, 4);
+audio_rechannel!(audio_rechannel_8_5, 8, 5);
+audio_rechannel!(audio_rechannel_8_6, 8, 6);
+audio_rechannel!(audio_rechannel_8_7, 8, 7);
+
 pub fn test_nat_type() {
     let mut i = 0;
     std::thread::spawn(move || loop {

src/server/audio_service.rs

@@ -140,21 +140,28 @@ mod cpal_impl {
     }
 
     fn send(
-        data: &[f32],
+        data: Vec<f32>,
         sample_rate0: u32,
         sample_rate: u32,
-        channels: u16,
+        device_channel: u16,
+        encode_channel: u16,
         encoder: &mut Encoder,
         sp: &GenericService,
     ) {
-        let buffer;
+        let mut data = data;
-        let data = if sample_rate0 != sample_rate {
+        if sample_rate0 != sample_rate {
-            buffer = crate::common::resample_channels(data, sample_rate0, sample_rate, channels);
+            data = crate::common::audio_resample(&data, sample_rate0, sample_rate, device_channel);
-            &buffer
+        }
-        } else {
+        if device_channel != encode_channel {
-            data
+            data = crate::common::audio_rechannel(
-        };
+                data,
-        send_f32(data, encoder, sp);
+                sample_rate,
+                sample_rate,
+                device_channel,
+                encode_channel,
+            )
+        }
+        send_f32(&data, encoder, sp);
     }
 
     #[cfg(windows)]
@@ -230,23 +237,24 @@ mod cpal_impl {
         } else {
             48000
         };
+        let ch = if config.channels() > 1 { Stereo } else { Mono };
         let stream = match config.sample_format() {
-            I8 => build_input_stream::<i8>(device, &config, sp, sample_rate)?,
+            I8 => build_input_stream::<i8>(device, &config, sp, sample_rate, ch)?,
-            I16 => build_input_stream::<i16>(device, &config, sp, sample_rate)?,
+            I16 => build_input_stream::<i16>(device, &config, sp, sample_rate, ch)?,
-            I32 => build_input_stream::<i32>(device, &config, sp, sample_rate)?,
+            I32 => build_input_stream::<i32>(device, &config, sp, sample_rate, ch)?,
-            I64 => build_input_stream::<i64>(device, &config, sp, sample_rate)?,
+            I64 => build_input_stream::<i64>(device, &config, sp, sample_rate, ch)?,
-            U8 => build_input_stream::<u8>(device, &config, sp, sample_rate)?,
+            U8 => build_input_stream::<u8>(device, &config, sp, sample_rate, ch)?,
-            U16 => build_input_stream::<u16>(device, &config, sp, sample_rate)?,
+            U16 => build_input_stream::<u16>(device, &config, sp, sample_rate, ch)?,
-            U32 => build_input_stream::<u32>(device, &config, sp, sample_rate)?,
+            U32 => build_input_stream::<u32>(device, &config, sp, sample_rate, ch)?,
-            U64 => build_input_stream::<u64>(device, &config, sp, sample_rate)?,
+            U64 => build_input_stream::<u64>(device, &config, sp, sample_rate, ch)?,
-            F32 => build_input_stream::<f32>(device, &config, sp, sample_rate)?,
+            F32 => build_input_stream::<f32>(device, &config, sp, sample_rate, ch)?,
-            F64 => build_input_stream::<f64>(device, &config, sp, sample_rate)?,
+            F64 => build_input_stream::<f64>(device, &config, sp, sample_rate, ch)?,
             f => bail!("unsupported audio format: {:?}", f),
         };
         stream.play()?;
         Ok((
             Box::new(stream),
-            Arc::new(create_format_msg(sample_rate, config.channels())),
+            Arc::new(create_format_msg(sample_rate, ch as _)),
         ))
     }
 
@@ -255,6 +263,7 @@ mod cpal_impl {
         config: &cpal::SupportedStreamConfig,
         sp: GenericService,
         sample_rate: u32,
+        encode_channel: magnum_opus::Channels,
     ) -> ResultType<cpal::Stream>
     where
         T: cpal::SizedSample + dasp::sample::ToSample<f32>,
@@ -268,19 +277,17 @@ mod cpal_impl {
         unsafe {
             AUDIO_ZERO_COUNT = 0;
         }
-        let mut encoder = Encoder::new(
+        let device_channel = config.channels();
-            sample_rate,
+        let mut encoder = Encoder::new(sample_rate, encode_channel, LowDelay)?;
-            if config.channels() > 1 { Stereo } else { Mono },
-            LowDelay,
-        )?;
-        let channels = config.channels();
         // https://www.opus-codec.org/docs/html_api/group__opusencoder.html#gace941e4ef26ed844879fde342ffbe546
         // https://chromium.googlesource.com/chromium/deps/opus/+/1.1.1/include/opus.h
-        let encode_len = sample_rate as usize * channels as usize / 100; // 10 ms
+        let frame_size = sample_rate as usize / 100; // 10 ms
+        let encode_len = frame_size * encode_channel as usize;
+        let rechannel_len = encode_len * device_channel as usize / encode_channel as usize;
         INPUT_BUFFER.lock().unwrap().clear();
         let timeout = None;
         let stream_config = StreamConfig {
-            channels,
+            channels: device_channel,
             sample_rate: config.sample_rate(),
             buffer_size: BufferSize::Default,
         };
@@ -290,13 +297,14 @@ mod cpal_impl {
                 let buffer: Vec<f32> = data.iter().map(|s| T::to_sample(*s)).collect();
                 let mut lock = INPUT_BUFFER.lock().unwrap();
                 lock.extend(buffer);
-                while lock.len() >= encode_len {
+                while lock.len() >= rechannel_len {
-                    let frame: Vec<f32> = lock.drain(0..encode_len).collect();
+                    let frame: Vec<f32> = lock.drain(0..rechannel_len).collect();
                     send(
-                        &frame,
+                        frame,
                         sample_rate_0,
                         sample_rate,
-                        channels,
+                        device_channel,
+                        encode_channel as _,
                         &mut encoder,
                         &sp,
                     );