mirror/rustdesk
1
0
Fork 0
mirror of https://github.com/rustdesk/rustdesk.git synced 2024-11-23 19:49:05 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

audio rechannel

Browse Source 
Signed-off-by: 21pages <pages21@163.com>
This commit is contained in:
21pages 2023-04-20 17:26:17 +08:00
parent 94d7339457
commit 0d8fe1b7b8
5 changed files with 227 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Cargo.lock generated
View File

@ -2083,6 +2083,15 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "fon"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad46a0e6c9bc688823a742aa969b5c08fdc56c2a436ee00d5c6fbcb5982c55c4"
dependencies = [
"libm",
]
[[package]]
name = "foreign-types"
version = "0.3.2"
@ -3293,6 +3302,12 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "libm"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
[[package]]
name = "libpulse-binding"
version = "2.27.1"
@ -5124,6 +5139,7 @@ dependencies = [
"evdev",
"flutter_rust_bridge",
"flutter_rust_bridge_codegen",
"fon",
"fruitbasket",
"hbb_common",
"hex",

1
Cargo.toml
View File

@ -71,6 +71,7 @@ reqwest = { version = "0.11", features = ["blocking", "json", "rustls-tls"], def
chrono = "0.4"
cidr-utils = "0.5"
libloading = "0.7"
fon = "0.6"
[target.'cfg(not(any(target_os = "android", target_os = "linux")))'.dependencies]
cpal = "0.15"

24
src/client.rs
View File

@ -753,6 +753,7 @@ pub struct AudioHandler {
#[cfg(not(any(target_os = "android", target_os = "linux")))]
audio_stream: Option<Box<dyn StreamTrait>>,
channels: u16,
device_channel: u16,
#[cfg(not(any(target_os = "android", target_os = "linux")))]
ready: Arc<std::sync::Mutex<bool>>,
}
@ -825,8 +826,8 @@ impl AudioHandler {
let sample_format = config.sample_format();
log::info!("Default output format: {:?}", config);
log::info!("Remote input format: {:?}", format0);
let mut config: StreamConfig = config.into();
config.channels = format0.channels as _;
let config: StreamConfig = config.into();
self.device_channel = config.channels;
match sample_format {
cpal::SampleFormat::I8 => self.build_output_stream::<i8>(&config, &device)?,
cpal::SampleFormat::I16 => self.build_output_stream::<i16>(&config, &device)?,
@ -884,20 +885,25 @@ impl AudioHandler {
let sample_rate0 = self.sample_rate.0;
let sample_rate = self.sample_rate.1;
let audio_buffer = self.audio_buffer.0.clone();
let mut buffer = buffer[0..n].to_owned();
if sample_rate != sample_rate0 {
let buffer = crate::resample_channels(
buffer = crate::audio_resample(
&buffer[0..n],
sample_rate0,
sample_rate,
channels,
);
audio_buffer.lock().unwrap().push_slice_overwrite(&buffer);
} else {
audio_buffer
.lock()
.unwrap()
.push_slice_overwrite(&buffer[0..n]);
}
if self.channels != self.device_channel {
buffer = crate::audio_rechannel(
buffer,
sample_rate,
sample_rate,
self.channels,
self.device_channel,
);
}
audio_buffer.lock().unwrap().push_slice_overwrite(&buffer);
}
#[cfg(target_os = "android")]
{

156
src/common.rs
View File

@ -273,7 +273,7 @@ pub fn resample_channels(
}
#[cfg(feature = "use_dasp")]
pub fn resample_channels(
pub fn audio_resample(
data: &[f32],
sample_rate0: u32,
sample_rate: u32,
@ -309,7 +309,7 @@ pub fn resample_channels(
}
#[cfg(feature = "use_samplerate")]
pub fn resample_channels(
pub fn audio_resample(
data: &[f32],
sample_rate0: u32,
sample_rate: u32,
@ -326,6 +326,158 @@ pub fn resample_channels(
.unwrap_or_default()
}
pub fn audio_rechannel(
input: Vec<f32>,
in_hz: u32,
out_hz: u32,
in_chan: u16,
output_chan: u16,
) -> Vec<f32> {
if in_chan == output_chan {
return input;
}
let mut input = input;
input.truncate(input.len() / in_chan as usize * in_chan as usize);
match (in_chan, output_chan) {
(1, 2) => audio_rechannel_1_2(&input, in_hz, out_hz),
(1, 3) => audio_rechannel_1_3(&input, in_hz, out_hz),
(1, 4) => audio_rechannel_1_4(&input, in_hz, out_hz),
(1, 5) => audio_rechannel_1_5(&input, in_hz, out_hz),
(1, 6) => audio_rechannel_1_6(&input, in_hz, out_hz),
(1, 7) => audio_rechannel_1_7(&input, in_hz, out_hz),
(1, 8) => audio_rechannel_1_8(&input, in_hz, out_hz),
(2, 1) => audio_rechannel_2_1(&input, in_hz, out_hz),
(2, 3) => audio_rechannel_2_3(&input, in_hz, out_hz),
(2, 4) => audio_rechannel_2_4(&input, in_hz, out_hz),
(2, 5) => audio_rechannel_2_5(&input, in_hz, out_hz),
(2, 6) => audio_rechannel_2_6(&input, in_hz, out_hz),
(2, 7) => audio_rechannel_2_7(&input, in_hz, out_hz),
(2, 8) => audio_rechannel_2_8(&input, in_hz, out_hz),
(3, 1) => audio_rechannel_3_1(&input, in_hz, out_hz),
(3, 2) => audio_rechannel_3_2(&input, in_hz, out_hz),
(3, 4) => audio_rechannel_3_4(&input, in_hz, out_hz),
(3, 5) => audio_rechannel_3_5(&input, in_hz, out_hz),
(3, 6) => audio_rechannel_3_6(&input, in_hz, out_hz),
(3, 7) => audio_rechannel_3_7(&input, in_hz, out_hz),
(3, 8) => audio_rechannel_3_8(&input, in_hz, out_hz),
(4, 1) => audio_rechannel_4_1(&input, in_hz, out_hz),
(4, 2) => audio_rechannel_4_2(&input, in_hz, out_hz),
(4, 3) => audio_rechannel_4_3(&input, in_hz, out_hz),
(4, 5) => audio_rechannel_4_5(&input, in_hz, out_hz),
(4, 6) => audio_rechannel_4_6(&input, in_hz, out_hz),
(4, 7) => audio_rechannel_4_7(&input, in_hz, out_hz),
(4, 8) => audio_rechannel_4_8(&input, in_hz, out_hz),
(5, 1) => audio_rechannel_5_1(&input, in_hz, out_hz),
(5, 2) => audio_rechannel_5_2(&input, in_hz, out_hz),
(5, 3) => audio_rechannel_5_3(&input, in_hz, out_hz),
(5, 4) => audio_rechannel_5_4(&input, in_hz, out_hz),
(5, 6) => audio_rechannel_5_6(&input, in_hz, out_hz),
(5, 7) => audio_rechannel_5_7(&input, in_hz, out_hz),
(5, 8) => audio_rechannel_5_8(&input, in_hz, out_hz),
(6, 1) => audio_rechannel_6_1(&input, in_hz, out_hz),
(6, 2) => audio_rechannel_6_2(&input, in_hz, out_hz),
(6, 3) => audio_rechannel_6_3(&input, in_hz, out_hz),
(6, 4) => audio_rechannel_6_4(&input, in_hz, out_hz),
(6, 5) => audio_rechannel_6_5(&input, in_hz, out_hz),
(6, 7) => audio_rechannel_6_7(&input, in_hz, out_hz),
(6, 8) => audio_rechannel_6_8(&input, in_hz, out_hz),
(7, 1) => audio_rechannel_7_1(&input, in_hz, out_hz),
(7, 2) => audio_rechannel_7_2(&input, in_hz, out_hz),
(7, 3) => audio_rechannel_7_3(&input, in_hz, out_hz),
(7, 4) => audio_rechannel_7_4(&input, in_hz, out_hz),
(7, 5) => audio_rechannel_7_5(&input, in_hz, out_hz),
(7, 6) => audio_rechannel_7_6(&input, in_hz, out_hz),
(7, 8) => audio_rechannel_7_8(&input, in_hz, out_hz),
(8, 1) => audio_rechannel_8_1(&input, in_hz, out_hz),
(8, 2) => audio_rechannel_8_2(&input, in_hz, out_hz),
(8, 3) => audio_rechannel_8_3(&input, in_hz, out_hz),
(8, 4) => audio_rechannel_8_4(&input, in_hz, out_hz),
(8, 5) => audio_rechannel_8_5(&input, in_hz, out_hz),
(8, 6) => audio_rechannel_8_6(&input, in_hz, out_hz),
(8, 7) => audio_rechannel_8_7(&input, in_hz, out_hz),
_ => input,
}
}
macro_rules! audio_rechannel {
($name:ident, $in_channels:expr, $out_channels:expr) => {
fn $name(input: &[f32], in_hz: u32, out_hz: u32) -> Vec<f32> {
use fon::{chan::Ch32, Audio, Frame};
let mut in_audio =
Audio::<Ch32, $in_channels>::with_silence(in_hz, input.len() / $in_channels);
for (x, y) in input.chunks_exact($in_channels).zip(in_audio.iter_mut()) {
let mut f = Frame::<Ch32, $in_channels>::default();
let mut i = 0;
for c in f.channels_mut() {
*c = x[i].into();
i += 1;
}
*y = f;
}
Audio::<Ch32, $out_channels>::with_audio(out_hz, &in_audio)
.as_f32_slice()
.to_owned()
}
};
}
audio_rechannel!(audio_rechannel_1_2, 1, 2);
audio_rechannel!(audio_rechannel_1_3, 1, 3);
audio_rechannel!(audio_rechannel_1_4, 1, 4);
audio_rechannel!(audio_rechannel_1_5, 1, 5);
audio_rechannel!(audio_rechannel_1_6, 1, 6);
audio_rechannel!(audio_rechannel_1_7, 1, 7);
audio_rechannel!(audio_rechannel_1_8, 1, 8);
audio_rechannel!(audio_rechannel_2_1, 2, 1);
audio_rechannel!(audio_rechannel_2_3, 2, 3);
audio_rechannel!(audio_rechannel_2_4, 2, 4);
audio_rechannel!(audio_rechannel_2_5, 2, 5);
audio_rechannel!(audio_rechannel_2_6, 2, 6);
audio_rechannel!(audio_rechannel_2_7, 2, 7);
audio_rechannel!(audio_rechannel_2_8, 2, 8);
audio_rechannel!(audio_rechannel_3_1, 3, 1);
audio_rechannel!(audio_rechannel_3_2, 3, 2);
audio_rechannel!(audio_rechannel_3_4, 3, 4);
audio_rechannel!(audio_rechannel_3_5, 3, 5);
audio_rechannel!(audio_rechannel_3_6, 3, 6);
audio_rechannel!(audio_rechannel_3_7, 3, 7);
audio_rechannel!(audio_rechannel_3_8, 3, 8);
audio_rechannel!(audio_rechannel_4_1, 4, 1);
audio_rechannel!(audio_rechannel_4_2, 4, 2);
audio_rechannel!(audio_rechannel_4_3, 4, 3);
audio_rechannel!(audio_rechannel_4_5, 4, 5);
audio_rechannel!(audio_rechannel_4_6, 4, 6);
audio_rechannel!(audio_rechannel_4_7, 4, 7);
audio_rechannel!(audio_rechannel_4_8, 4, 8);
audio_rechannel!(audio_rechannel_5_1, 5, 1);
audio_rechannel!(audio_rechannel_5_2, 5, 2);
audio_rechannel!(audio_rechannel_5_3, 5, 3);
audio_rechannel!(audio_rechannel_5_4, 5, 4);
audio_rechannel!(audio_rechannel_5_6, 5, 6);
audio_rechannel!(audio_rechannel_5_7, 5, 7);
audio_rechannel!(audio_rechannel_5_8, 5, 8);
audio_rechannel!(audio_rechannel_6_1, 6, 1);
audio_rechannel!(audio_rechannel_6_2, 6, 2);
audio_rechannel!(audio_rechannel_6_3, 6, 3);
audio_rechannel!(audio_rechannel_6_4, 6, 4);
audio_rechannel!(audio_rechannel_6_5, 6, 5);
audio_rechannel!(audio_rechannel_6_7, 6, 7);
audio_rechannel!(audio_rechannel_6_8, 6, 8);
audio_rechannel!(audio_rechannel_7_1, 7, 1);
audio_rechannel!(audio_rechannel_7_2, 7, 2);
audio_rechannel!(audio_rechannel_7_3, 7, 3);
audio_rechannel!(audio_rechannel_7_4, 7, 4);
audio_rechannel!(audio_rechannel_7_5, 7, 5);
audio_rechannel!(audio_rechannel_7_6, 7, 6);
audio_rechannel!(audio_rechannel_7_8, 7, 8);
audio_rechannel!(audio_rechannel_8_1, 8, 1);
audio_rechannel!(audio_rechannel_8_2, 8, 2);
audio_rechannel!(audio_rechannel_8_3, 8, 3);
audio_rechannel!(audio_rechannel_8_4, 8, 4);
audio_rechannel!(audio_rechannel_8_5, 8, 5);
audio_rechannel!(audio_rechannel_8_6, 8, 6);
audio_rechannel!(audio_rechannel_8_7, 8, 7);
pub fn test_nat_type() {
let mut i = 0;
std::thread::spawn(move || loop {

74
src/server/audio_service.rs
View File

@ -140,21 +140,28 @@ mod cpal_impl {
}
fn send(
data: &[f32],
data: Vec<f32>,
sample_rate0: u32,
sample_rate: u32,
channels: u16,
device_channel: u16,
encode_channel: u16,
encoder: &mut Encoder,
sp: &GenericService,
) {
let buffer;
let data = if sample_rate0 != sample_rate {
buffer = crate::common::resample_channels(data, sample_rate0, sample_rate, channels);
&buffer
} else {
data
};
send_f32(data, encoder, sp);
let mut data = data;
if sample_rate0 != sample_rate {
data = crate::common::audio_resample(&data, sample_rate0, sample_rate, device_channel);
}
if device_channel != encode_channel {
data = crate::common::audio_rechannel(
data,
sample_rate,
sample_rate,
device_channel,
encode_channel,
)
}
send_f32(&data, encoder, sp);
}
#[cfg(windows)]
@ -230,23 +237,24 @@ mod cpal_impl {
} else {
48000
};
let ch = if config.channels() > 1 { Stereo } else { Mono };
let stream = match config.sample_format() {
I8 => build_input_stream::<i8>(device, &config, sp, sample_rate)?,
I16 => build_input_stream::<i16>(device, &config, sp, sample_rate)?,
I32 => build_input_stream::<i32>(device, &config, sp, sample_rate)?,
I64 => build_input_stream::<i64>(device, &config, sp, sample_rate)?,
U8 => build_input_stream::<u8>(device, &config, sp, sample_rate)?,
U16 => build_input_stream::<u16>(device, &config, sp, sample_rate)?,
U32 => build_input_stream::<u32>(device, &config, sp, sample_rate)?,
U64 => build_input_stream::<u64>(device, &config, sp, sample_rate)?,
F32 => build_input_stream::<f32>(device, &config, sp, sample_rate)?,
F64 => build_input_stream::<f64>(device, &config, sp, sample_rate)?,
I8 => build_input_stream::<i8>(device, &config, sp, sample_rate, ch)?,
I16 => build_input_stream::<i16>(device, &config, sp, sample_rate, ch)?,
I32 => build_input_stream::<i32>(device, &config, sp, sample_rate, ch)?,
I64 => build_input_stream::<i64>(device, &config, sp, sample_rate, ch)?,
U8 => build_input_stream::<u8>(device, &config, sp, sample_rate, ch)?,
U16 => build_input_stream::<u16>(device, &config, sp, sample_rate, ch)?,
U32 => build_input_stream::<u32>(device, &config, sp, sample_rate, ch)?,
U64 => build_input_stream::<u64>(device, &config, sp, sample_rate, ch)?,
F32 => build_input_stream::<f32>(device, &config, sp, sample_rate, ch)?,
F64 => build_input_stream::<f64>(device, &config, sp, sample_rate, ch)?,
f => bail!("unsupported audio format: {:?}", f),
};
stream.play()?;
Ok((
Box::new(stream),
Arc::new(create_format_msg(sample_rate, config.channels())),
Arc::new(create_format_msg(sample_rate, ch as _)),
))
}
@ -255,6 +263,7 @@ mod cpal_impl {
config: &cpal::SupportedStreamConfig,
sp: GenericService,
sample_rate: u32,
encode_channel: magnum_opus::Channels,
) -> ResultType<cpal::Stream>
where
T: cpal::SizedSample + dasp::sample::ToSample<f32>,
@ -268,19 +277,17 @@ mod cpal_impl {
unsafe {
AUDIO_ZERO_COUNT = 0;
}
let mut encoder = Encoder::new(
sample_rate,
if config.channels() > 1 { Stereo } else { Mono },
LowDelay,
)?;
let channels = config.channels();
let device_channel = config.channels();
let mut encoder = Encoder::new(sample_rate, encode_channel, LowDelay)?;
// https://www.opus-codec.org/docs/html_api/group__opusencoder.html#gace941e4ef26ed844879fde342ffbe546
// https://chromium.googlesource.com/chromium/deps/opus/+/1.1.1/include/opus.h
let encode_len = sample_rate as usize * channels as usize / 100; // 10 ms
let frame_size = sample_rate as usize / 100; // 10 ms
let encode_len = frame_size * encode_channel as usize;
let rechannel_len = encode_len * device_channel as usize / encode_channel as usize;
INPUT_BUFFER.lock().unwrap().clear();
let timeout = None;
let stream_config = StreamConfig {
channels,
channels: device_channel,
sample_rate: config.sample_rate(),
buffer_size: BufferSize::Default,
};
@ -290,13 +297,14 @@ mod cpal_impl {
let buffer: Vec<f32> = data.iter().map(|s| T::to_sample(*s)).collect();
let mut lock = INPUT_BUFFER.lock().unwrap();
lock.extend(buffer);
while lock.len() >= encode_len {
let frame: Vec<f32> = lock.drain(0..encode_len).collect();
while lock.len() >= rechannel_len {
let frame: Vec<f32> = lock.drain(0..rechannel_len).collect();
send(
&frame,
frame,
sample_rate_0,
sample_rate,
channels,
device_channel,
encode_channel as _,
&mut encoder,
&sp,
);