From 70968638bf15557c67454458a6ab8cf94f49c34b Mon Sep 17 00:00:00 2001 From: 21pages Date: Sun, 29 May 2022 17:23:14 +0800 Subject: [PATCH 01/12] scrap: add hwcodec Signed-off-by: 21pages --- Cargo.lock | 12 + libs/hbb_common/protos/message.proto | 31 +++ libs/scrap/Cargo.toml | 4 + libs/scrap/examples/record-screen.rs | 29 ++- libs/scrap/src/common/codec.rs | 322 ++++++++++++++++----------- libs/scrap/src/common/coder.rs | 276 +++++++++++++++++++++++ libs/scrap/src/common/convert.rs | 211 ++++++++++++++++++ libs/scrap/src/common/hwcodec.rs | 290 ++++++++++++++++++++++++ libs/scrap/src/common/mod.rs | 8 +- src/client.rs | 47 ++-- src/server/connection.rs | 15 ++ src/server/video_service.rs | 89 +++----- 12 files changed, 1112 insertions(+), 222 deletions(-) create mode 100644 libs/scrap/src/common/coder.rs create mode 100644 libs/scrap/src/common/hwcodec.rs diff --git a/Cargo.lock b/Cargo.lock index e6942ef72..35df55da9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2233,6 +2233,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hwcodec" +version = "0.1.0" +source = "git+https://github.com/21pages/hwcodec#373d55d38c23cc8a9ef9961b3b2979d5fc9d1bc4" +dependencies = [ + "bindgen", + "cc", + "log", +] + [[package]] name = "hyper" version = "0.14.18" @@ -4257,6 +4267,8 @@ dependencies = [ "gstreamer", "gstreamer-app", "gstreamer-video", + "hbb_common", + "hwcodec", "jni", "lazy_static", "libc", diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto index c296e6a7f..5c4a0944d 100644 --- a/libs/hbb_common/protos/message.proto +++ b/libs/hbb_common/protos/message.proto @@ -9,6 +9,26 @@ message VP9 { message VP9s { repeated VP9 frames = 1; } +message H264 { + bytes data = 1; + bool key = 2; + int64 pts = 3; +} + +message H264s { + repeated H264 h264s = 1; +} + +message H265 { + bytes data = 1; + bool key = 2; + int64 pts = 3; +} + +message H265s { + repeated H265 h265s = 1; +} + message RGB { bool compress = 1; } // planes data send directly in binary for better use arraybuffer on web @@ -22,6 +42,8 @@ message VideoFrame { VP9s vp9s = 6; RGB rgb = 7; YUV yuv = 8; + H264s h264s = 10; + H265s h265s = 11; } int64 timestamp = 9; } @@ -425,6 +447,14 @@ enum ImageQuality { Best = 4; } +message VideoCodecState { + int32 ScoreVpx = 1; + bool H264 = 2; + int32 ScoreH264 = 3; + bool H265 = 4; + int32 ScoreH265 = 5; +} + message OptionMessage { enum BoolOption { NotSet = 0; @@ -440,6 +470,7 @@ message OptionMessage { BoolOption disable_audio = 7; BoolOption disable_clipboard = 8; BoolOption enable_file_transfer = 9; + VideoCodecState video_codec_state = 10; } message OptionResponse { diff --git a/libs/scrap/Cargo.toml b/libs/scrap/Cargo.toml index 00c4509ab..b72ada14d 100644 --- a/libs/scrap/Cargo.toml +++ b/libs/scrap/Cargo.toml @@ -17,6 +17,7 @@ block = "0.1" cfg-if = "1.0" libc = "0.2" num_cpus = "1.13" +hbb_common = { path = "../hbb_common" } [dependencies.winapi] version = "0.3" @@ -46,3 +47,6 @@ tracing = { version = "0.1", optional = true } gstreamer = { version = "0.16", optional = true } gstreamer-app = { version = "0.16", features = ["v1_10"], optional = true } gstreamer-video = { version = "0.16", optional = true } + +[target.'cfg(target_os = "windows")'.dependencies] +hwcodec = { git = "https://github.com/21pages/hwcodec", optional = true } diff --git a/libs/scrap/examples/record-screen.rs b/libs/scrap/examples/record-screen.rs index 2a56c0dcd..035ad587e 100644 --- a/libs/scrap/examples/record-screen.rs +++ b/libs/scrap/examples/record-screen.rs @@ -13,6 +13,7 @@ use std::time::{Duration, Instant}; use std::{io, thread}; use docopt::Docopt; +use scrap::coder::{EncoderApi, EncoderCfg}; use webm::mux; use webm::mux::Track; @@ -89,27 +90,25 @@ fn main() -> io::Result<()> { mux::Segment::new(mux::Writer::new(out)).expect("Could not initialize the multiplexer."); let (vpx_codec, mux_codec) = match args.flag_codec { - Codec::Vp8 => (vpx_encode::VideoCodecId::VP8, mux::VideoCodecId::VP8), - Codec::Vp9 => (vpx_encode::VideoCodecId::VP9, mux::VideoCodecId::VP9), + Codec::Vp8 => (vpx_encode::VpxVideoCodecId::VP8, mux::VideoCodecId::VP8), + Codec::Vp9 => (vpx_encode::VpxVideoCodecId::VP9, mux::VideoCodecId::VP9), }; let mut vt = webm.add_video_track(width, height, None, mux_codec); // Setup the encoder. - let mut vpx = vpx_encode::Encoder::new( - &vpx_encode::Config { - width, - height, - timebase: [1, 1000], - bitrate: args.flag_bv, - codec: vpx_codec, - rc_min_quantizer: 0, - rc_max_quantizer: 0, - speed: 6, - }, - 0, - ) + let mut vpx = vpx_encode::VpxEncoder::new(EncoderCfg::VPX(vpx_encode::VpxEncoderConfig { + width, + height, + timebase: [1, 1000], + bitrate: args.flag_bv, + codec: vpx_codec, + rc_min_quantizer: 0, + rc_max_quantizer: 0, + speed: 6, + num_threads: 0, + })) .unwrap(); // Start recording. diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index f1533d7cf..59bab099f 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -2,29 +2,36 @@ // https://github.com/astraw/env-libvpx-sys // https://github.com/rust-av/vpx-rs/blob/master/src/decoder.rs +use hbb_common::anyhow::{anyhow, Context}; +use hbb_common::message_proto::{Message, VP9s, VideoFrame, VP9}; +use hbb_common::ResultType; + +use crate::coder::EncoderApi; +use crate::STRIDE_ALIGN; + use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *}; use std::os::raw::{c_int, c_uint}; use std::{ptr, slice}; #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VideoCodecId { +pub enum VpxVideoCodecId { VP8, VP9, } -impl Default for VideoCodecId { - fn default() -> VideoCodecId { - VideoCodecId::VP9 +impl Default for VpxVideoCodecId { + fn default() -> VpxVideoCodecId { + VpxVideoCodecId::VP9 } } -pub struct Encoder { +pub struct VpxEncoder { ctx: vpx_codec_ctx_t, width: usize, height: usize, } -pub struct Decoder { +pub struct VpxDecoder { ctx: vpx_codec_ctx_t, } @@ -82,118 +89,152 @@ macro_rules! call_vpx_ptr { }}; } -impl Encoder { - pub fn new(config: &Config, num_threads: u32) -> Result { - let i; - if cfg!(feature = "VP8") { - i = match config.codec { - VideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_cx()), - VideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_cx()), - }; - } else { - i = call_vpx_ptr!(vpx_codec_vp9_cx()); +impl EncoderApi for VpxEncoder { + fn new(cfg: crate::coder::EncoderCfg) -> ResultType + where + Self: Sized, + { + match cfg { + crate::coder::EncoderCfg::VPX(config) => { + let i; + if cfg!(feature = "VP8") { + i = match config.codec { + VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_cx()), + VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_cx()), + }; + } else { + i = call_vpx_ptr!(vpx_codec_vp9_cx()); + } + let mut c = unsafe { std::mem::MaybeUninit::zeroed().assume_init() }; + call_vpx!(vpx_codec_enc_config_default(i, &mut c, 0)); + + // https://www.webmproject.org/docs/encoder-parameters/ + // default: c.rc_min_quantizer = 0, c.rc_max_quantizer = 63 + // try rc_resize_allowed later + + c.g_w = config.width; + c.g_h = config.height; + c.g_timebase.num = config.timebase[0]; + c.g_timebase.den = config.timebase[1]; + c.rc_target_bitrate = config.bitrate; + c.rc_undershoot_pct = 95; + c.rc_dropframe_thresh = 25; + if config.rc_min_quantizer > 0 { + c.rc_min_quantizer = config.rc_min_quantizer; + } + if config.rc_max_quantizer > 0 { + c.rc_max_quantizer = config.rc_max_quantizer; + } + let mut speed = config.speed; + if speed <= 0 { + speed = 6; + } + + c.g_threads = if config.num_threads == 0 { + num_cpus::get() as _ + } else { + config.num_threads + }; + c.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT; + // https://developers.google.com/media/vp9/bitrate-modes/ + // Constant Bitrate mode (CBR) is recommended for live streaming with VP9. + c.rc_end_usage = vpx_rc_mode::VPX_CBR; + // c.kf_min_dist = 0; + // c.kf_max_dist = 999999; + c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot + + /* + VPX encoder支持two-pass encode，这是为了rate control的。 + 对于两遍编码，就是需要整个编码过程做两次，第一次会得到一些新的控制参数来进行第二遍的编码， + 这样可以在相同的bitrate下得到最好的PSNR + */ + + let mut ctx = Default::default(); + call_vpx!(vpx_codec_enc_init_ver( + &mut ctx, + i, + &c, + 0, + VPX_ENCODER_ABI_VERSION as _ + )); + + if config.codec == VpxVideoCodecId::VP9 { + // set encoder internal speed settings + // in ffmpeg, it is --speed option + /* + set to 0 or a positive value 1-16, the codec will try to adapt its + complexity depending on the time it spends encoding. Increasing this + number will make the speed go up and the quality go down. + Negative values mean strict enforcement of this + while positive values are adaptive + */ + /* https://developers.google.com/media/vp9/live-encoding + Speed 5 to 8 should be used for live / real-time encoding. + Lower numbers (5 or 6) are higher quality but require more CPU power. + Higher numbers (7 or 8) will be lower quality but more manageable for lower latency + use cases and also for lower CPU power devices such as mobile. + */ + call_vpx!(vpx_codec_control_(&mut ctx, VP8E_SET_CPUUSED as _, speed,)); + // set row level multi-threading + /* + as some people in comments and below have already commented, + more recent versions of libvpx support -row-mt 1 to enable tile row + multi-threading. This can increase the number of tiles by up to 4x in VP9 + (since the max number of tile rows is 4, regardless of video height). + To enable this, use -tile-rows N where N is the number of tile rows in + log2 units (so -tile-rows 1 means 2 tile rows and -tile-rows 2 means 4 tile + rows). The total number of active threads will then be equal to + $tile_rows * $tile_columns + */ + call_vpx!(vpx_codec_control_( + &mut ctx, + VP9E_SET_ROW_MT as _, + 1 as c_int + )); + + call_vpx!(vpx_codec_control_( + &mut ctx, + VP9E_SET_TILE_COLUMNS as _, + 4 as c_int + )); + } + + Ok(Self { + ctx, + width: config.width as _, + height: config.height as _, + }) + } + _ => Err(anyhow!("encoder type mismatch")), } - let mut c = unsafe { std::mem::MaybeUninit::zeroed().assume_init() }; - call_vpx!(vpx_codec_enc_config_default(i, &mut c, 0)); - - // https://www.webmproject.org/docs/encoder-parameters/ - // default: c.rc_min_quantizer = 0, c.rc_max_quantizer = 63 - // try rc_resize_allowed later - - c.g_w = config.width; - c.g_h = config.height; - c.g_timebase.num = config.timebase[0]; - c.g_timebase.den = config.timebase[1]; - c.rc_target_bitrate = config.bitrate; - c.rc_undershoot_pct = 95; - c.rc_dropframe_thresh = 25; - if config.rc_min_quantizer > 0 { - c.rc_min_quantizer = config.rc_min_quantizer; - } - if config.rc_max_quantizer > 0 { - c.rc_max_quantizer = config.rc_max_quantizer; - } - let mut speed = config.speed; - if speed <= 0 { - speed = 6; - } - - c.g_threads = if num_threads == 0 { - num_cpus::get() as _ - } else { - num_threads - }; - c.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT; - // https://developers.google.com/media/vp9/bitrate-modes/ - // Constant Bitrate mode (CBR) is recommended for live streaming with VP9. - c.rc_end_usage = vpx_rc_mode::VPX_CBR; - // c.kf_min_dist = 0; - // c.kf_max_dist = 999999; - c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot - - /* - VPX encoder支持two-pass encode，这是为了rate control的。 - 对于两遍编码，就是需要整个编码过程做两次，第一次会得到一些新的控制参数来进行第二遍的编码， - 这样可以在相同的bitrate下得到最好的PSNR - */ - - let mut ctx = Default::default(); - call_vpx!(vpx_codec_enc_init_ver( - &mut ctx, - i, - &c, - 0, - VPX_ENCODER_ABI_VERSION as _ - )); - - if config.codec == VideoCodecId::VP9 { - // set encoder internal speed settings - // in ffmpeg, it is --speed option - /* - set to 0 or a positive value 1-16, the codec will try to adapt its - complexity depending on the time it spends encoding. Increasing this - number will make the speed go up and the quality go down. - Negative values mean strict enforcement of this - while positive values are adaptive - */ - /* https://developers.google.com/media/vp9/live-encoding - Speed 5 to 8 should be used for live / real-time encoding. - Lower numbers (5 or 6) are higher quality but require more CPU power. - Higher numbers (7 or 8) will be lower quality but more manageable for lower latency - use cases and also for lower CPU power devices such as mobile. - */ - call_vpx!(vpx_codec_control_(&mut ctx, VP8E_SET_CPUUSED as _, speed,)); - // set row level multi-threading - /* - as some people in comments and below have already commented, - more recent versions of libvpx support -row-mt 1 to enable tile row - multi-threading. This can increase the number of tiles by up to 4x in VP9 - (since the max number of tile rows is 4, regardless of video height). - To enable this, use -tile-rows N where N is the number of tile rows in - log2 units (so -tile-rows 1 means 2 tile rows and -tile-rows 2 means 4 tile - rows). The total number of active threads will then be equal to - $tile_rows * $tile_columns - */ - call_vpx!(vpx_codec_control_( - &mut ctx, - VP9E_SET_ROW_MT as _, - 1 as c_int - )); - - call_vpx!(vpx_codec_control_( - &mut ctx, - VP9E_SET_TILE_COLUMNS as _, - 4 as c_int - )); - } - - Ok(Self { - ctx, - width: config.width as _, - height: config.height as _, - }) } + fn encode_to_message(&mut self, frame: &[u8], ms: i64) -> ResultType { + let mut frames = Vec::new(); + for ref frame in self + .encode(ms, frame, STRIDE_ALIGN) + .with_context(|| "Failed to encode")? + { + frames.push(VpxEncoder::create_frame(frame)); + } + for ref frame in self.flush().with_context(|| "Failed to flush")? { + frames.push(VpxEncoder::create_frame(frame)); + } + + // to-do: flush periodically, e.g. 1 second + if frames.len() > 0 { + Ok(VpxEncoder::create_msg(frames)) + } else { + Err(anyhow!("no valid frame")) + } + } + + fn use_yuv(&self) -> bool { + true + } +} + +impl VpxEncoder { pub fn encode(&mut self, pts: i64, data: &[u8], stride_align: usize) -> Result { assert!(2 * data.len() >= 3 * self.width * self.height); @@ -238,9 +279,31 @@ impl Encoder { iter: ptr::null(), }) } + + #[inline] + fn create_msg(vp9s: Vec) -> Message { + let mut msg_out = Message::new(); + let mut vf = VideoFrame::new(); + vf.set_vp9s(VP9s { + frames: vp9s.into(), + ..Default::default() + }); + msg_out.set_video_frame(vf); + msg_out + } + + #[inline] + fn create_frame(frame: &EncodeFrame) -> VP9 { + VP9 { + data: frame.data.to_vec(), + key: frame.key, + pts: frame.pts, + ..Default::default() + } + } } -impl Drop for Encoder { +impl Drop for VpxEncoder { fn drop(&mut self) { unsafe { let result = vpx_codec_destroy(&mut self.ctx); @@ -262,7 +325,7 @@ pub struct EncodeFrame<'a> { } #[derive(Clone, Copy, Debug)] -pub struct Config { +pub struct VpxEncoderConfig { /// The width (in pixels). pub width: c_uint, /// The height (in pixels). @@ -272,10 +335,17 @@ pub struct Config { /// The target bitrate (in kilobits per second). pub bitrate: c_uint, /// The codec - pub codec: VideoCodecId, + pub codec: VpxVideoCodecId, pub rc_min_quantizer: u32, pub rc_max_quantizer: u32, pub speed: i32, + pub num_threads: u32, +} + +#[derive(Clone, Copy, Debug)] +pub struct VpxDecoderConfig { + pub codec: VpxVideoCodecId, + pub num_threads: u32, } pub struct EncodeFrames<'a> { @@ -306,31 +376,31 @@ impl<'a> Iterator for EncodeFrames<'a> { } } -impl Decoder { +impl VpxDecoder { /// Create a new decoder /// /// # Errors /// /// The function may fail if the underlying libvpx does not provide /// the VP9 decoder. - pub fn new(codec: VideoCodecId, num_threads: u32) -> Result { + pub fn new(config: VpxDecoderConfig) -> Result { // This is sound because `vpx_codec_ctx` is a repr(C) struct without any field that can // cause UB if uninitialized. let i; if cfg!(feature = "VP8") { - i = match codec { - VideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_dx()), - VideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_dx()), + i = match config.codec { + VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_dx()), + VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_dx()), }; } else { i = call_vpx_ptr!(vpx_codec_vp9_dx()); } let mut ctx = Default::default(); let cfg = vpx_codec_dec_cfg_t { - threads: if num_threads == 0 { + threads: if config.num_threads == 0 { num_cpus::get() as _ } else { - num_threads + config.num_threads }, w: 0, h: 0, @@ -405,7 +475,7 @@ impl Decoder { } } -impl Drop for Decoder { +impl Drop for VpxDecoder { fn drop(&mut self) { unsafe { let result = vpx_codec_destroy(&mut self.ctx); diff --git a/libs/scrap/src/common/coder.rs b/libs/scrap/src/common/coder.rs new file mode 100644 index 000000000..b90a9c3d7 --- /dev/null +++ b/libs/scrap/src/common/coder.rs @@ -0,0 +1,276 @@ +use std::ops::{Deref, DerefMut}; +#[cfg(feature = "hwcodec")] +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +use crate::codec::*; +#[cfg(feature = "hwcodec")] +use crate::hwcodec::*; + +use hbb_common::{ + anyhow::anyhow, + message_proto::{video_frame, Message, VP9s, VideoCodecState}, + ResultType, +}; +#[cfg(feature = "hwcodec")] +use hbb_common::{ + lazy_static, log, + message_proto::{H264s, H265s}, +}; + +#[cfg(feature = "hwcodec")] +lazy_static::lazy_static! { + static ref VIDEO_CODEC_STATES: Arc>> = Default::default(); +} + +#[derive(Debug, Clone)] +pub struct HwEncoderConfig { + pub codec_name: String, + pub fps: i32, + pub width: usize, + pub height: usize, +} + +pub enum EncoderCfg { + VPX(VpxEncoderConfig), + HW(HwEncoderConfig), +} + +pub trait EncoderApi { + fn new(cfg: EncoderCfg) -> ResultType + where + Self: Sized; + + fn encode_to_message(&mut self, frame: &[u8], ms: i64) -> ResultType; + + fn use_yuv(&self) -> bool; +} + +pub struct DecoderCfg { + pub vpx: VpxDecoderConfig, +} + +pub struct Encoder { + pub codec: Box, +} + +impl Deref for Encoder { + type Target = Box; + + fn deref(&self) -> &Self::Target { + &self.codec + } +} + +impl DerefMut for Encoder { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.codec + } +} + +pub struct Decoder { + vpx: VpxDecoder, + #[cfg(feature = "hwcodec")] + hw: Arc>, + #[cfg(feature = "hwcodec")] + i420: Vec, +} + +impl Encoder { + pub fn new(config: EncoderCfg) -> ResultType { + match config { + EncoderCfg::VPX(_) => Ok(Encoder { + codec: Box::new(VpxEncoder::new(config)?), + }), + + #[cfg(feature = "hwcodec")] + EncoderCfg::HW(_) => Ok(Encoder { + codec: Box::new(HwEncoder::new(config)?), + }), + #[cfg(not(feature = "hwcodec"))] + _ => Err(anyhow!("unsupported encoder type")), + } + } + + // TODO + pub fn update_video_encoder(id: i32, decoder: Option) { + #[cfg(feature = "hwcodec")] + { + let mut states = VIDEO_CODEC_STATES.lock().unwrap(); + match decoder { + Some(decoder) => states.insert(id, decoder), + None => states.remove(&id), + }; + let (encoder_h264, encoder_h265) = HwEncoder::best(); + let mut enabled_h264 = encoder_h264.is_some(); + let mut enabled_h265 = encoder_h265.is_some(); + let mut score_vpx = 90; + let mut score_h264 = encoder_h264.as_ref().map_or(0, |c| c.score); + let mut score_h265 = encoder_h265.as_ref().map_or(0, |c| c.score); + + for state in states.iter() { + enabled_h264 = enabled_h264 && state.1.H264; + enabled_h265 = enabled_h265 && state.1.H265; + score_vpx += state.1.ScoreVpx; + score_h264 += state.1.ScoreH264; + score_h265 += state.1.ScoreH265; + } + + let current_encoder_name = HwEncoder::current_name(); + if enabled_h265 && score_h265 >= score_vpx && score_h265 >= score_h264 { + *current_encoder_name.lock().unwrap() = Some(encoder_h265.unwrap().name); + } else if enabled_h264 && score_h264 >= score_vpx && score_h264 >= score_h265 { + *current_encoder_name.lock().unwrap() = Some(encoder_h264.unwrap().name); + } else { + *current_encoder_name.lock().unwrap() = None; + } + if states.len() > 0 { + log::info!( + "connection count:{}, h264:{}, h265:{}, score: vpx({}), h264({}), h265({}), set current encoder name {:?}", + states.len(), + enabled_h264, + enabled_h265, + score_vpx, + score_h264, + score_h265, + current_encoder_name.lock().unwrap() + ) + } + } + #[cfg(not(feature = "hwcodec"))] + { + let _ = id; + let _ = decoder; + } + } + #[inline] + pub fn current_hw_encoder_name() -> Option { + #[cfg(feature = "hwcodec")] + return HwEncoder::current_name().lock().unwrap().clone(); + #[cfg(not(feature = "hwcodec"))] + return None; + } +} + +impl Decoder { + // TODO + pub fn video_codec_state() -> VideoCodecState { + let mut state = VideoCodecState::default(); + state.ScoreVpx = 90; + + #[cfg(feature = "hwcodec")] + { + let hw = HwDecoder::instance(); + state.H264 = hw.lock().unwrap().h264.is_some(); + state.ScoreH264 = hw.lock().unwrap().h264.as_ref().map_or(0, |d| d.info.score); + state.H265 = hw.lock().unwrap().h265.is_some(); + state.ScoreH265 = hw.lock().unwrap().h265.as_ref().map_or(0, |d| d.info.score); + } + + state + } + + pub fn new(config: DecoderCfg) -> Decoder { + let vpx = VpxDecoder::new(config.vpx).unwrap(); + Decoder { + vpx, + #[cfg(feature = "hwcodec")] + hw: HwDecoder::instance(), + #[cfg(feature = "hwcodec")] + i420: vec![], + } + } + + pub fn handle_video_frame( + &mut self, + frame: &video_frame::Union, + rgb: &mut Vec, + ) -> ResultType { + match frame { + video_frame::Union::vp9s(vp9s) => { + Decoder::handle_vp9s_video_frame(&mut self.vpx, vp9s, rgb) + } + #[cfg(feature = "hwcodec")] + video_frame::Union::h264s(h264s) => { + if let Some(decoder) = &mut self.hw.lock().unwrap().h264 { + Decoder::handle_h264s_video_frame(decoder, h264s, rgb, &mut self.i420) + } else { + Err(anyhow!("don't support h264!")) + } + } + #[cfg(feature = "hwcodec")] + video_frame::Union::h265s(h265s) => { + if let Some(decoder) = &mut self.hw.lock().unwrap().h265 { + Decoder::handle_h265s_video_frame(decoder, h265s, rgb, &mut self.i420) + } else { + Err(anyhow!("don't support h265!")) + } + } + _ => Err(anyhow!("unsupported video frame type!")), + } + } + + fn handle_vp9s_video_frame( + decoder: &mut VpxDecoder, + vp9s: &VP9s, + rgb: &mut Vec, + ) -> ResultType { + let mut last_frame = Image::new(); + for vp9 in vp9s.frames.iter() { + for frame in decoder.decode(&vp9.data)? { + drop(last_frame); + last_frame = frame; + } + } + for frame in decoder.flush()? { + drop(last_frame); + last_frame = frame; + } + if last_frame.is_null() { + Ok(false) + } else { + last_frame.rgb(1, true, rgb); + Ok(true) + } + } + + #[cfg(feature = "hwcodec")] + fn handle_h264s_video_frame( + decoder: &mut HwDecoder, + h264s: &H264s, + rgb: &mut Vec, + i420: &mut Vec, + ) -> ResultType { + let mut ret = false; + for h264 in h264s.h264s.iter() { + for image in decoder.decode(&h264.data)? { + // TODO: just process the last frame + if image.bgra(rgb, i420).is_ok() { + ret = true; + } + } + } + return Ok(ret); + } + + #[cfg(feature = "hwcodec")] + fn handle_h265s_video_frame( + decoder: &mut HwDecoder, + h265s: &H265s, + rgb: &mut Vec, + i420: &mut Vec, + ) -> ResultType { + let mut ret = false; + for h265 in h265s.h265s.iter() { + for image in decoder.decode(&h265.data)? { + // TODO: just process the last frame + if image.bgra(rgb, i420).is_ok() { + ret = true; + } + } + } + return Ok(ret); + } +} diff --git a/libs/scrap/src/common/convert.rs b/libs/scrap/src/common/convert.rs index 1e5f2164d..306a217ea 100644 --- a/libs/scrap/src/common/convert.rs +++ b/libs/scrap/src/common/convert.rs @@ -49,6 +49,17 @@ extern "C" { height: c_int, ) -> c_int; + pub fn ARGBToNV12( + src_bgra: *const u8, + src_stride_bgra: c_int, + dst_y: *mut u8, + dst_stride_y: c_int, + dst_uv: *mut u8, + dst_stride_uv: c_int, + width: c_int, + height: c_int, + ) -> c_int; + pub fn NV12ToI420( src_y: *const u8, src_stride_y: c_int, @@ -91,6 +102,17 @@ extern "C" { width: c_int, height: c_int, ) -> c_int; + + pub fn NV12ToARGB( + src_y: *const u8, + src_stride_y: c_int, + src_uv: *const u8, + src_stride_uv: c_int, + dst_rgba: *mut u8, + dst_stride_rgba: c_int, + width: c_int, + height: c_int, + ) -> c_int; } // https://github.com/webmproject/libvpx/blob/master/vpx/src/vpx_image.c @@ -220,3 +242,192 @@ pub unsafe fn nv12_to_i420( height as _, ); } + +#[cfg(feature = "hwcodec")] +pub mod hw { + use hbb_common::{anyhow::anyhow, ResultType}; + use hwcodec::{ffmpeg::ffmpeg_linesize_offset_length, AVPixelFormat}; + + pub fn hw_bgra_to_i420( + width: usize, + height: usize, + stride: &[i32], + offset: &[i32], + length: i32, + src: &[u8], + dst: &mut Vec, + ) { + let stride_y = stride[0] as usize; + let stride_u = stride[1] as usize; + let stride_v = stride[2] as usize; + let offset_u = offset[0] as usize; + let offset_v = offset[1] as usize; + + dst.resize(length as _, 0); + let dst_y = dst.as_mut_ptr(); + let dst_u = dst[offset_u..].as_mut_ptr(); + let dst_v = dst[offset_v..].as_mut_ptr(); + unsafe { + super::ARGBToI420( + src.as_ptr(), + (src.len() / height) as _, + dst_y, + stride_y as _, + dst_u, + stride_u as _, + dst_v, + stride_v as _, + width as _, + height as _, + ); + } + } + + pub fn hw_bgra_to_nv12( + width: usize, + height: usize, + stride: &[i32], + offset: &[i32], + length: i32, + src: &[u8], + dst: &mut Vec, + ) { + let stride_y = stride[0] as usize; + let stride_uv = stride[1] as usize; + let offset_uv = offset[0] as usize; + + dst.resize(length as _, 0); + let dst_y = dst.as_mut_ptr(); + let dst_uv = dst[offset_uv..].as_mut_ptr(); + unsafe { + super::ARGBToNV12( + src.as_ptr(), + (src.len() / height) as _, + dst_y, + stride_y as _, + dst_uv, + stride_uv as _, + width as _, + height as _, + ); + } + } + + #[cfg(target_os = "windows")] + pub fn hw_nv12_to_bgra( + width: usize, + height: usize, + src_y: &[u8], + src_uv: &[u8], + src_stride_y: usize, + src_stride_uv: usize, + dst: &mut Vec, + i420: &mut Vec, + align: usize, + ) -> ResultType<()> { + let nv12_stride_y = src_stride_y; + let nv12_stride_uv = src_stride_uv; + if let Ok((linesize_i420, offset_i420, i420_len)) = + ffmpeg_linesize_offset_length(AVPixelFormat::AV_PIX_FMT_YUV420P, width, height, align) + { + dst.resize(width * height * 4, 0); + let i420_stride_y = linesize_i420[0]; + let i420_stride_u = linesize_i420[1]; + let i420_stride_v = linesize_i420[2]; + i420.resize(i420_len as _, 0); + + unsafe { + let i420_offset_y = i420.as_ptr().add(0) as _; + let i420_offset_u = i420.as_ptr().add(offset_i420[0] as _) as _; + let i420_offset_v = i420.as_ptr().add(offset_i420[1] as _) as _; + super::NV12ToI420( + src_y.as_ptr(), + nv12_stride_y as _, + src_uv.as_ptr(), + nv12_stride_uv as _, + i420_offset_y, + i420_stride_y, + i420_offset_u, + i420_stride_u, + i420_offset_v, + i420_stride_v, + width as _, + height as _, + ); + super::I420ToARGB( + i420_offset_y, + i420_stride_y, + i420_offset_u, + i420_stride_u, + i420_offset_v, + i420_stride_v, + dst.as_mut_ptr(), + (width * 4) as _, + width as _, + height as _, + ); + return Ok(()); + }; + } + return Err(anyhow!("get linesize offset failed")); + } + + #[cfg(not(target_os = "windows"))] + pub fn hw_nv12_to_bgra( + width: usize, + height: usize, + src_y: &[u8], + src_uv: &[u8], + src_stride_y: usize, + src_stride_uv: usize, + dst: &mut Vec, + ) -> ResultType<()> { + dst.resize(width * height * 4, 0); + unsafe { + match super::NV12ToARGB( + src_y.as_ptr(), + src_stride_y as _, + src_uv.as_ptr(), + src_stride_uv as _, + dst.as_mut_ptr(), + (width * 4) as _, + width as _, + height as _, + ) { + 0 => Ok(()), + _ => Err(anyhow!("NV12ToARGB failed")), + } + } + } + + pub fn hw_i420_to_bgra( + width: usize, + height: usize, + src_y: &[u8], + src_u: &[u8], + src_v: &[u8], + src_stride_y: usize, + src_stride_u: usize, + src_stride_v: usize, + dst: &mut Vec, + ) { + let src_y = src_y.as_ptr(); + let src_u = src_u.as_ptr(); + let src_v = src_v.as_ptr(); + dst.resize(width * height * 4, 0); + unsafe { + super::I420ToARGB( + src_y, + src_stride_y as _, + src_u, + src_stride_u as _, + src_v, + src_stride_v as _, + dst.as_mut_ptr(), + (width * 4) as _, + width as _, + height as _, + ); + }; + } +} diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs new file mode 100644 index 000000000..f2867c592 --- /dev/null +++ b/libs/scrap/src/common/hwcodec.rs @@ -0,0 +1,290 @@ +use crate::{ + coder::{EncoderApi, EncoderCfg}, + hw, HW_STRIDE_ALIGN, +}; +use hbb_common::{ + anyhow::{anyhow, Context}, + lazy_static, log, + message_proto::{H264s, H265s, Message, VideoFrame, H264, H265}, + ResultType, +}; +use hwcodec::{ + decode::{DecodeContext, DecodeFrame, Decoder}, + encode::{EncodeContext, EncodeFrame, Encoder}, + ffmpeg::{CodecInfo, DataFormat}, + AVPixelFormat, +}; +use std::sync::{Arc, Mutex, Once}; + +lazy_static::lazy_static! { + static ref HW_ENCODER_NAME: Arc>> = Default::default(); + static ref HW_DECODER_INSTANCE: Arc> = Arc::new(Mutex::new(HwDecoderInstance { + h264: None, + h265: None, + })); +} + +const DEFAULT_PIXFMT: AVPixelFormat = AVPixelFormat::AV_PIX_FMT_YUV420P; + +pub struct HwEncoder { + encoder: Encoder, + yuv: Vec, + pub format: DataFormat, + pub pixfmt: AVPixelFormat, +} + +impl EncoderApi for HwEncoder { + fn new(cfg: EncoderCfg) -> ResultType + where + Self: Sized, + { + match cfg { + EncoderCfg::HW(config) => { + let ctx = EncodeContext { + name: config.codec_name.clone(), + fps: config.fps as _, + width: config.width as _, + height: config.height as _, + pixfmt: DEFAULT_PIXFMT, + align: HW_STRIDE_ALIGN as _, + }; + let format = match Encoder::format_from_name(config.codec_name.clone()) { + Ok(format) => format, + Err(_) => { + return Err(anyhow!(format!( + "failed to get format from name:{}", + config.codec_name + ))) + } + }; + match Encoder::new(ctx.clone()) { + Ok(encoder) => Ok(HwEncoder { + encoder, + yuv: vec![], + format, + pixfmt: ctx.pixfmt, + }), + Err(_) => Err(anyhow!(format!("Failed to create encoder"))), + } + } + _ => Err(anyhow!("encoder type mismatch")), + } + } + + fn encode_to_message( + &mut self, + frame: &[u8], + _ms: i64, + ) -> ResultType { + let mut msg_out = Message::new(); + let mut vf = VideoFrame::new(); + match self.format { + DataFormat::H264 => { + let mut h264s = Vec::new(); + for frame in self.encode(frame).with_context(|| "Failed to encode")? { + h264s.push(H264 { + data: frame.data, + pts: frame.pts as _, + ..Default::default() + }); + } + if h264s.len() > 0 { + vf.set_h264s(H264s { + h264s: h264s.into(), + ..Default::default() + }); + msg_out.set_video_frame(vf); + Ok(msg_out) + } else { + Err(anyhow!("no valid frame")) + } + } + DataFormat::H265 => { + let mut h265s = Vec::new(); + for frame in self.encode(frame).with_context(|| "Failed to encode")? { + h265s.push(H265 { + data: frame.data, + pts: frame.pts, + ..Default::default() + }); + } + if h265s.len() > 0 { + vf.set_h265s(H265s { + h265s, + ..Default::default() + }); + msg_out.set_video_frame(vf); + Ok(msg_out) + } else { + Err(anyhow!("no valid frame")) + } + } + } + } + + fn use_yuv(&self) -> bool { + false + } +} + +impl HwEncoder { + pub fn best() -> (Option, Option) { + let ctx = EncodeContext { + name: String::from(""), + fps: 30, + width: 1920, + height: 1080, + pixfmt: DEFAULT_PIXFMT, + align: HW_STRIDE_ALIGN as _, + }; + CodecInfo::score(Encoder::avaliable_encoders(ctx)) + } + + pub fn current_name() -> Arc>> { + HW_ENCODER_NAME.clone() + } + + pub fn encode(&mut self, bgra: &[u8]) -> ResultType> { + match self.pixfmt { + AVPixelFormat::AV_PIX_FMT_YUV420P => hw::hw_bgra_to_i420( + self.encoder.ctx.width as _, + self.encoder.ctx.height as _, + &self.encoder.linesize, + &self.encoder.offset, + self.encoder.length, + bgra, + &mut self.yuv, + ), + AVPixelFormat::AV_PIX_FMT_NV12 => hw::hw_bgra_to_nv12( + self.encoder.ctx.width as _, + self.encoder.ctx.height as _, + &self.encoder.linesize, + &self.encoder.offset, + self.encoder.length, + bgra, + &mut self.yuv, + ), + } + + match self.encoder.encode(&self.yuv) { + Ok(v) => { + let mut data = Vec::::new(); + data.append(v); + Ok(data) + } + Err(_) => Ok(Vec::::new()), + } + } +} + +pub struct HwDecoder { + decoder: Decoder, + pub info: CodecInfo, +} + +pub struct HwDecoderInstance { + pub h264: Option, + pub h265: Option, +} + +impl HwDecoder { + pub fn instance() -> Arc> { + static ONCE: Once = Once::new(); + // TODO: different process + ONCE.call_once(|| { + let avaliable = Decoder::avaliable_decoders(); + let mut decoders = vec![]; + for decoder in avaliable { + if let Ok(d) = HwDecoder::new(decoder) { + decoders.push(d); + } + } + + let mut h264: Option = None; + let mut h265: Option = None; + for decoder in decoders { + match decoder.info.format { + DataFormat::H264 => match &h264 { + Some(old) => { + if decoder.info.score > old.info.score { + h264 = Some(decoder) + } + } + None => h264 = Some(decoder), + }, + DataFormat::H265 => match &h265 { + Some(old) => { + if decoder.info.score > old.info.score { + h265 = Some(decoder) + } + } + None => h265 = Some(decoder), + }, + } + } + if h264.is_some() { + log::info!("h264 decoder:{:?}", h264.as_ref().unwrap().info); + } + if h265.is_some() { + log::info!("h265 decoder:{:?}", h265.as_ref().unwrap().info); + } + HW_DECODER_INSTANCE.lock().unwrap().h264 = h264; + HW_DECODER_INSTANCE.lock().unwrap().h265 = h265; + }); + HW_DECODER_INSTANCE.clone() + } + + pub fn new(info: CodecInfo) -> ResultType { + let ctx = DecodeContext { + name: info.name.clone(), + device_type: info.hwdevice.clone(), + }; + match Decoder::new(ctx) { + Ok(decoder) => Ok(HwDecoder { decoder, info }), + Err(_) => Err(anyhow!(format!("Failed to create decoder"))), + } + } + pub fn decode(&mut self, data: &[u8]) -> ResultType> { + match self.decoder.decode(data) { + Ok(v) => Ok(v.iter().map(|f| HwDecoderImage { frame: f }).collect()), + Err(_) => Ok(vec![]), + } + } +} + +pub struct HwDecoderImage<'a> { + frame: &'a DecodeFrame, +} + +impl HwDecoderImage<'_> { + pub fn bgra(&self, bgra: &mut Vec, i420: &mut Vec) -> ResultType<()> { + let frame = self.frame; + match frame.pixfmt { + AVPixelFormat::AV_PIX_FMT_NV12 => hw::hw_nv12_to_bgra( + frame.width as _, + frame.height as _, + &frame.data[0], + &frame.data[1], + frame.linesize[0] as _, + frame.linesize[1] as _, + bgra, + i420, + HW_STRIDE_ALIGN, + ), + AVPixelFormat::AV_PIX_FMT_YUV420P => { + hw::hw_i420_to_bgra( + frame.width as _, + frame.height as _, + &frame.data[0], + &frame.data[1], + &frame.data[2], + frame.linesize[0] as _, + frame.linesize[1] as _, + frame.linesize[2] as _, + bgra, + ); + return Ok(()); + } + } + } +} diff --git a/libs/scrap/src/common/mod.rs b/libs/scrap/src/common/mod.rs index dd2b4295a..662be28af 100644 --- a/libs/scrap/src/common/mod.rs +++ b/libs/scrap/src/common/mod.rs @@ -28,17 +28,19 @@ cfg_if! { } pub mod codec; +pub mod coder; mod convert; +#[cfg(feature = "hwcodec")] +pub mod hwcodec; pub use self::convert::*; pub const STRIDE_ALIGN: usize = 64; // commonly used in libvpx vpx_img_alloc caller +pub const HW_STRIDE_ALIGN: usize = 0; // recommended by av_frame_get_buffer mod vpx; #[inline] pub fn would_block_if_equal(old: &mut Vec, b: &[u8]) -> std::io::Result<()> { - let b = unsafe { - std::slice::from_raw_parts::(b.as_ptr() as _, b.len() / 16) - }; + let b = unsafe { std::slice::from_raw_parts::(b.as_ptr() as _, b.len() / 16) }; if b == &old[..] { return Err(std::io::ErrorKind::WouldBlock.into()); } diff --git a/src/client.rs b/src/client.rs index be2b788ab..915878774 100644 --- a/src/client.rs +++ b/src/client.rs @@ -12,6 +12,11 @@ use cpal::{ Device, Host, StreamConfig, }; use magnum_opus::{Channels::*, Decoder as AudioDecoder}; +use scrap::{ + coder::{Decoder, DecoderCfg}, + VpxDecoderConfig, VpxVideoCodecId, +}; + use sha2::{Digest, Sha256}; use uuid::Uuid; @@ -30,7 +35,6 @@ use hbb_common::{ tokio::time::Duration, AddrMangle, ResultType, Stream, }; -use scrap::{Decoder, Image, VideoCodecId}; pub use super::lang::*; pub mod file_trait; @@ -717,7 +721,12 @@ pub struct VideoHandler { impl VideoHandler { pub fn new(latency_controller: Arc>) -> Self { VideoHandler { - decoder: Decoder::new(VideoCodecId::VP9, (num_cpus::get() / 2) as _).unwrap(), + decoder: Decoder::new(DecoderCfg { + vpx: VpxDecoderConfig { + codec: VpxVideoCodecId::VP9, + num_threads: (num_cpus::get() / 2) as _, + }, + }), latency_controller, rgb: Default::default(), } @@ -731,33 +740,18 @@ impl VideoHandler { .update_video(vf.timestamp); } match &vf.union { - Some(video_frame::Union::vp9s(vp9s)) => self.handle_vp9s(vp9s), + Some(frame) => self.decoder.handle_video_frame(frame, &mut self.rgb), _ => Ok(false), } } - pub fn handle_vp9s(&mut self, vp9s: &VP9s) -> ResultType { - let mut last_frame = Image::new(); - for vp9 in vp9s.frames.iter() { - for frame in self.decoder.decode(&vp9.data)? { - drop(last_frame); - last_frame = frame; - } - } - for frame in self.decoder.flush()? { - drop(last_frame); - last_frame = frame; - } - if last_frame.is_null() { - Ok(false) - } else { - last_frame.rgb(1, true, &mut self.rgb); - Ok(true) - } - } - pub fn reset(&mut self) { - self.decoder = Decoder::new(VideoCodecId::VP9, 1).unwrap(); + self.decoder = Decoder::new(DecoderCfg { + vpx: VpxDecoderConfig { + codec: VpxVideoCodecId::VP9, + num_threads: 1, + }, + }); } } @@ -951,6 +945,11 @@ impl LoginConfigHandler { msg.disable_clipboard = BoolOption::Yes.into(); n += 1; } + // TODO: add option + let state = Decoder::video_codec_state(); + msg.video_codec_state = hbb_common::protobuf::MessageField::some(state); + n += 1; + if n > 0 { Some(msg) } else { diff --git a/src/server/connection.rs b/src/server/connection.rs index 3a026d924..fea7c6605 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -365,6 +365,7 @@ impl Connection { video_service::notify_video_frame_feched(id, None); super::video_service::update_test_latency(id, 0); super::video_service::update_image_quality(id, None); + scrap::coder::Encoder::update_video_encoder(id, None); if let Err(err) = conn.try_port_forward_loop(&mut rx_from_cm).await { conn.on_close(&err.to_string(), false); } @@ -1186,6 +1187,20 @@ impl Connection { } } } + + // TODO: add option + if let Some(q) = o.video_codec_state.clone().take() { + scrap::coder::Encoder::update_video_encoder(self.inner.id(), Some(q)); + } else { + scrap::coder::Encoder::update_video_encoder( + self.inner.id(), + Some(VideoCodecState { + H264: false, + H265: false, + ..Default::default() + }), + ); + } } fn on_close(&mut self, reason: &str, lock: bool) { diff --git a/src/server/video_service.rs b/src/server/video_service.rs index 17b545426..225df306a 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -26,7 +26,11 @@ use hbb_common::tokio::{ Mutex as TokioMutex, }, }; -use scrap::{Capturer, Config, Display, EncodeFrame, Encoder, VideoCodecId, STRIDE_ALIGN}; +use scrap::{ + codec::{VpxEncoderConfig, VpxVideoCodecId}, + coder::{Encoder, EncoderCfg, HwEncoderConfig}, + Capturer, Display, +}; use std::{ collections::HashSet, io::ErrorKind::WouldBlock, @@ -172,27 +176,39 @@ fn run(sp: GenericService) -> ResultType<()> { num_cpus::get_physical(), num_cpus::get(), ); - // Capturer object is expensive, avoiding to create it frequently. - let mut c = Capturer::new(display, true).with_context(|| "Failed to create capturer")?; let q = get_image_quality(); let (bitrate, rc_min_quantizer, rc_max_quantizer, speed) = get_quality(width, height, q); log::info!("bitrate={}, rc_min_quantizer={}", bitrate, rc_min_quantizer); - let cfg = Config { - width: width as _, - height: height as _, - timebase: [1, 1000], // Output timestamp precision - bitrate, - codec: VideoCodecId::VP9, - rc_min_quantizer, - rc_max_quantizer, - speed, + + let encoder_cfg = match Encoder::current_hw_encoder_name() { + Some(codec_name) => EncoderCfg::HW(HwEncoderConfig { + codec_name, + fps, + width, + height, + }), + None => EncoderCfg::VPX(VpxEncoderConfig { + width: width as _, + height: height as _, + timebase: [1, 1000], // Output timestamp precision + bitrate, + codec: VpxVideoCodecId::VP9, + rc_min_quantizer, + rc_max_quantizer, + speed, + num_threads: (num_cpus::get() / 2) as _, + }), }; - let mut vpx; - match Encoder::new(&cfg, (num_cpus::get() / 2) as _) { - Ok(x) => vpx = x, + + let mut encoder; + match Encoder::new(encoder_cfg) { + Ok(x) => encoder = x, Err(err) => bail!("Failed to create encoder: {}", err), } + // Capturer object is expensive, avoiding to create it frequently. + let mut c = + Capturer::new(display, encoder.use_yuv()).with_context(|| "Failed to create capturer")?; if *SWITCH.lock().unwrap() { log::debug!("Broadcasting display switch"); @@ -277,7 +293,7 @@ fn run(sp: GenericService) -> ResultType<()> { Ok(frame) => { let time = now - start; let ms = (time.as_secs() * 1000 + time.subsec_millis() as u64) as i64; - let send_conn_ids = handle_one_frame(&sp, &frame, ms, &mut vpx)?; + let send_conn_ids = handle_one_frame(&sp, &frame, ms, &mut encoder)?; frame_controller.set_send(now, send_conn_ids); #[cfg(windows)] { @@ -333,35 +349,12 @@ fn run(sp: GenericService) -> ResultType<()> { Ok(()) } -#[inline] -fn create_msg(vp9s: Vec) -> Message { - let mut msg_out = Message::new(); - let mut vf = VideoFrame::new(); - vf.set_vp9s(VP9s { - frames: vp9s.into(), - ..Default::default() - }); - vf.timestamp = crate::common::get_time(); - msg_out.set_video_frame(vf); - msg_out -} - -#[inline] -fn create_frame(frame: &EncodeFrame) -> VP9 { - VP9 { - data: frame.data.to_vec(), - key: frame.key, - pts: frame.pts, - ..Default::default() - } -} - #[inline] fn handle_one_frame( sp: &GenericService, frame: &[u8], ms: i64, - vpx: &mut Encoder, + encoder: &mut Encoder, ) -> ResultType> { sp.snapshot(|sps| { // so that new sub and old sub share the same encoder after switch @@ -372,20 +365,8 @@ fn handle_one_frame( })?; let mut send_conn_ids: HashSet = Default::default(); - let mut frames = Vec::new(); - for ref frame in vpx - .encode(ms, frame, STRIDE_ALIGN) - .with_context(|| "Failed to encode")? - { - frames.push(create_frame(frame)); - } - for ref frame in vpx.flush().with_context(|| "Failed to flush")? { - frames.push(create_frame(frame)); - } - - // to-do: flush periodically, e.g. 1 second - if frames.len() > 0 { - send_conn_ids = sp.send_video_frame(create_msg(frames)); + if let Ok(msg) = encoder.encode_to_message(frame, ms) { + send_conn_ids = sp.send_video_frame(msg); } Ok(send_conn_ids) } From 6677fc9b3022f4ed86f69e8aeda414202ef3c428 Mon Sep 17 00:00:00 2001 From: 21pages Date: Sun, 29 May 2022 18:16:35 +0800 Subject: [PATCH 02/12] scrap: rename codec.rs to vpxcodec.rs Signed-off-by: 21pages --- libs/scrap/examples/record-screen.rs | 2 +- libs/scrap/src/common/coder.rs | 2 +- libs/scrap/src/common/mod.rs | 4 +- .../src/common/{codec.rs => vpxcodec.rs} | 1212 ++++++++--------- src/server/video_service.rs | 2 +- 5 files changed, 611 insertions(+), 611 deletions(-) rename libs/scrap/src/common/{codec.rs => vpxcodec.rs} (96%) diff --git a/libs/scrap/examples/record-screen.rs b/libs/scrap/examples/record-screen.rs index 035ad587e..8be7dc132 100644 --- a/libs/scrap/examples/record-screen.rs +++ b/libs/scrap/examples/record-screen.rs @@ -17,7 +17,7 @@ use scrap::coder::{EncoderApi, EncoderCfg}; use webm::mux; use webm::mux::Track; -use scrap::codec as vpx_encode; +use scrap::vpxcodec as vpx_encode; use scrap::{Capturer, Display, STRIDE_ALIGN}; const USAGE: &'static str = " diff --git a/libs/scrap/src/common/coder.rs b/libs/scrap/src/common/coder.rs index b90a9c3d7..af2412c0f 100644 --- a/libs/scrap/src/common/coder.rs +++ b/libs/scrap/src/common/coder.rs @@ -5,9 +5,9 @@ use std::{ sync::{Arc, Mutex}, }; -use crate::codec::*; #[cfg(feature = "hwcodec")] use crate::hwcodec::*; +use crate::vpxcodec::*; use hbb_common::{ anyhow::anyhow, diff --git a/libs/scrap/src/common/mod.rs b/libs/scrap/src/common/mod.rs index 662be28af..993f20628 100644 --- a/libs/scrap/src/common/mod.rs +++ b/libs/scrap/src/common/mod.rs @@ -1,4 +1,4 @@ -pub use self::codec::*; +pub use self::vpxcodec::*; cfg_if! { if #[cfg(quartz)] { @@ -27,11 +27,11 @@ cfg_if! { } } -pub mod codec; pub mod coder; mod convert; #[cfg(feature = "hwcodec")] pub mod hwcodec; +pub mod vpxcodec; pub use self::convert::*; pub const STRIDE_ALIGN: usize = 64; // commonly used in libvpx vpx_img_alloc caller pub const HW_STRIDE_ALIGN: usize = 0; // recommended by av_frame_get_buffer diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/vpxcodec.rs similarity index 96% rename from libs/scrap/src/common/codec.rs rename to libs/scrap/src/common/vpxcodec.rs index 59bab099f..7301dc705 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/vpxcodec.rs @@ -1,606 +1,606 @@ -// https://github.com/astraw/vpx-encode -// https://github.com/astraw/env-libvpx-sys -// https://github.com/rust-av/vpx-rs/blob/master/src/decoder.rs - -use hbb_common::anyhow::{anyhow, Context}; -use hbb_common::message_proto::{Message, VP9s, VideoFrame, VP9}; -use hbb_common::ResultType; - -use crate::coder::EncoderApi; -use crate::STRIDE_ALIGN; - -use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *}; -use std::os::raw::{c_int, c_uint}; -use std::{ptr, slice}; - -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VpxVideoCodecId { - VP8, - VP9, -} - -impl Default for VpxVideoCodecId { - fn default() -> VpxVideoCodecId { - VpxVideoCodecId::VP9 - } -} - -pub struct VpxEncoder { - ctx: vpx_codec_ctx_t, - width: usize, - height: usize, -} - -pub struct VpxDecoder { - ctx: vpx_codec_ctx_t, -} - -#[derive(Debug)] -pub enum Error { - FailedCall(String), - BadPtr(String), -} - -impl std::fmt::Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::result::Result<(), std::fmt::Error> { - write!(f, "{:?}", self) - } -} - -impl std::error::Error for Error {} - -pub type Result = std::result::Result; - -macro_rules! call_vpx { - ($x:expr) => {{ - let result = unsafe { $x }; // original expression - let result_int = unsafe { std::mem::transmute::<_, i32>(result) }; - if result_int != 0 { - return Err(Error::FailedCall(format!( - "errcode={} {}:{}:{}:{}", - result_int, - module_path!(), - file!(), - line!(), - column!() - )) - .into()); - } - result - }}; -} - -macro_rules! call_vpx_ptr { - ($x:expr) => {{ - let result = unsafe { $x }; // original expression - let result_int = unsafe { std::mem::transmute::<_, isize>(result) }; - if result_int == 0 { - return Err(Error::BadPtr(format!( - "errcode={} {}:{}:{}:{}", - result_int, - module_path!(), - file!(), - line!(), - column!() - )) - .into()); - } - result - }}; -} - -impl EncoderApi for VpxEncoder { - fn new(cfg: crate::coder::EncoderCfg) -> ResultType - where - Self: Sized, - { - match cfg { - crate::coder::EncoderCfg::VPX(config) => { - let i; - if cfg!(feature = "VP8") { - i = match config.codec { - VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_cx()), - VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_cx()), - }; - } else { - i = call_vpx_ptr!(vpx_codec_vp9_cx()); - } - let mut c = unsafe { std::mem::MaybeUninit::zeroed().assume_init() }; - call_vpx!(vpx_codec_enc_config_default(i, &mut c, 0)); - - // https://www.webmproject.org/docs/encoder-parameters/ - // default: c.rc_min_quantizer = 0, c.rc_max_quantizer = 63 - // try rc_resize_allowed later - - c.g_w = config.width; - c.g_h = config.height; - c.g_timebase.num = config.timebase[0]; - c.g_timebase.den = config.timebase[1]; - c.rc_target_bitrate = config.bitrate; - c.rc_undershoot_pct = 95; - c.rc_dropframe_thresh = 25; - if config.rc_min_quantizer > 0 { - c.rc_min_quantizer = config.rc_min_quantizer; - } - if config.rc_max_quantizer > 0 { - c.rc_max_quantizer = config.rc_max_quantizer; - } - let mut speed = config.speed; - if speed <= 0 { - speed = 6; - } - - c.g_threads = if config.num_threads == 0 { - num_cpus::get() as _ - } else { - config.num_threads - }; - c.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT; - // https://developers.google.com/media/vp9/bitrate-modes/ - // Constant Bitrate mode (CBR) is recommended for live streaming with VP9. - c.rc_end_usage = vpx_rc_mode::VPX_CBR; - // c.kf_min_dist = 0; - // c.kf_max_dist = 999999; - c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot - - /* - VPX encoder支持two-pass encode，这是为了rate control的。 - 对于两遍编码，就是需要整个编码过程做两次，第一次会得到一些新的控制参数来进行第二遍的编码， - 这样可以在相同的bitrate下得到最好的PSNR - */ - - let mut ctx = Default::default(); - call_vpx!(vpx_codec_enc_init_ver( - &mut ctx, - i, - &c, - 0, - VPX_ENCODER_ABI_VERSION as _ - )); - - if config.codec == VpxVideoCodecId::VP9 { - // set encoder internal speed settings - // in ffmpeg, it is --speed option - /* - set to 0 or a positive value 1-16, the codec will try to adapt its - complexity depending on the time it spends encoding. Increasing this - number will make the speed go up and the quality go down. - Negative values mean strict enforcement of this - while positive values are adaptive - */ - /* https://developers.google.com/media/vp9/live-encoding - Speed 5 to 8 should be used for live / real-time encoding. - Lower numbers (5 or 6) are higher quality but require more CPU power. - Higher numbers (7 or 8) will be lower quality but more manageable for lower latency - use cases and also for lower CPU power devices such as mobile. - */ - call_vpx!(vpx_codec_control_(&mut ctx, VP8E_SET_CPUUSED as _, speed,)); - // set row level multi-threading - /* - as some people in comments and below have already commented, - more recent versions of libvpx support -row-mt 1 to enable tile row - multi-threading. This can increase the number of tiles by up to 4x in VP9 - (since the max number of tile rows is 4, regardless of video height). - To enable this, use -tile-rows N where N is the number of tile rows in - log2 units (so -tile-rows 1 means 2 tile rows and -tile-rows 2 means 4 tile - rows). The total number of active threads will then be equal to - $tile_rows * $tile_columns - */ - call_vpx!(vpx_codec_control_( - &mut ctx, - VP9E_SET_ROW_MT as _, - 1 as c_int - )); - - call_vpx!(vpx_codec_control_( - &mut ctx, - VP9E_SET_TILE_COLUMNS as _, - 4 as c_int - )); - } - - Ok(Self { - ctx, - width: config.width as _, - height: config.height as _, - }) - } - _ => Err(anyhow!("encoder type mismatch")), - } - } - - fn encode_to_message(&mut self, frame: &[u8], ms: i64) -> ResultType { - let mut frames = Vec::new(); - for ref frame in self - .encode(ms, frame, STRIDE_ALIGN) - .with_context(|| "Failed to encode")? - { - frames.push(VpxEncoder::create_frame(frame)); - } - for ref frame in self.flush().with_context(|| "Failed to flush")? { - frames.push(VpxEncoder::create_frame(frame)); - } - - // to-do: flush periodically, e.g. 1 second - if frames.len() > 0 { - Ok(VpxEncoder::create_msg(frames)) - } else { - Err(anyhow!("no valid frame")) - } - } - - fn use_yuv(&self) -> bool { - true - } -} - -impl VpxEncoder { - pub fn encode(&mut self, pts: i64, data: &[u8], stride_align: usize) -> Result { - assert!(2 * data.len() >= 3 * self.width * self.height); - - let mut image = Default::default(); - call_vpx_ptr!(vpx_img_wrap( - &mut image, - vpx_img_fmt::VPX_IMG_FMT_I420, - self.width as _, - self.height as _, - stride_align as _, - data.as_ptr() as _, - )); - - call_vpx!(vpx_codec_encode( - &mut self.ctx, - &image, - pts as _, - 1, // Duration - 0, // Flags - VPX_DL_REALTIME as _, - )); - - Ok(EncodeFrames { - ctx: &mut self.ctx, - iter: ptr::null(), - }) - } - - /// Notify the encoder to return any pending packets - pub fn flush(&mut self) -> Result { - call_vpx!(vpx_codec_encode( - &mut self.ctx, - ptr::null(), - -1, // PTS - 1, // Duration - 0, // Flags - VPX_DL_REALTIME as _, - )); - - Ok(EncodeFrames { - ctx: &mut self.ctx, - iter: ptr::null(), - }) - } - - #[inline] - fn create_msg(vp9s: Vec) -> Message { - let mut msg_out = Message::new(); - let mut vf = VideoFrame::new(); - vf.set_vp9s(VP9s { - frames: vp9s.into(), - ..Default::default() - }); - msg_out.set_video_frame(vf); - msg_out - } - - #[inline] - fn create_frame(frame: &EncodeFrame) -> VP9 { - VP9 { - data: frame.data.to_vec(), - key: frame.key, - pts: frame.pts, - ..Default::default() - } - } -} - -impl Drop for VpxEncoder { - fn drop(&mut self) { - unsafe { - let result = vpx_codec_destroy(&mut self.ctx); - if result != VPX_CODEC_OK { - panic!("failed to destroy vpx codec"); - } - } - } -} - -#[derive(Clone, Copy, Debug)] -pub struct EncodeFrame<'a> { - /// Compressed data. - pub data: &'a [u8], - /// Whether the frame is a keyframe. - pub key: bool, - /// Presentation timestamp (in timebase units). - pub pts: i64, -} - -#[derive(Clone, Copy, Debug)] -pub struct VpxEncoderConfig { - /// The width (in pixels). - pub width: c_uint, - /// The height (in pixels). - pub height: c_uint, - /// The timebase numerator and denominator (in seconds). - pub timebase: [c_int; 2], - /// The target bitrate (in kilobits per second). - pub bitrate: c_uint, - /// The codec - pub codec: VpxVideoCodecId, - pub rc_min_quantizer: u32, - pub rc_max_quantizer: u32, - pub speed: i32, - pub num_threads: u32, -} - -#[derive(Clone, Copy, Debug)] -pub struct VpxDecoderConfig { - pub codec: VpxVideoCodecId, - pub num_threads: u32, -} - -pub struct EncodeFrames<'a> { - ctx: &'a mut vpx_codec_ctx_t, - iter: vpx_codec_iter_t, -} - -impl<'a> Iterator for EncodeFrames<'a> { - type Item = EncodeFrame<'a>; - fn next(&mut self) -> Option { - loop { - unsafe { - let pkt = vpx_codec_get_cx_data(self.ctx, &mut self.iter); - if pkt.is_null() { - return None; - } else if (*pkt).kind == vpx_codec_cx_pkt_kind::VPX_CODEC_CX_FRAME_PKT { - let f = &(*pkt).data.frame; - return Some(Self::Item { - data: slice::from_raw_parts(f.buf as _, f.sz as _), - key: (f.flags & VPX_FRAME_IS_KEY) != 0, - pts: f.pts, - }); - } else { - // Ignore the packet. - } - } - } - } -} - -impl VpxDecoder { - /// Create a new decoder - /// - /// # Errors - /// - /// The function may fail if the underlying libvpx does not provide - /// the VP9 decoder. - pub fn new(config: VpxDecoderConfig) -> Result { - // This is sound because `vpx_codec_ctx` is a repr(C) struct without any field that can - // cause UB if uninitialized. - let i; - if cfg!(feature = "VP8") { - i = match config.codec { - VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_dx()), - VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_dx()), - }; - } else { - i = call_vpx_ptr!(vpx_codec_vp9_dx()); - } - let mut ctx = Default::default(); - let cfg = vpx_codec_dec_cfg_t { - threads: if config.num_threads == 0 { - num_cpus::get() as _ - } else { - config.num_threads - }, - w: 0, - h: 0, - }; - /* - unsafe { - println!("{}", vpx_codec_get_caps(i)); - } - */ - call_vpx!(vpx_codec_dec_init_ver( - &mut ctx, - i, - &cfg, - 0, - VPX_DECODER_ABI_VERSION as _, - )); - Ok(Self { ctx }) - } - - pub fn decode2rgb(&mut self, data: &[u8], rgba: bool) -> Result> { - let mut img = Image::new(); - for frame in self.decode(data)? { - drop(img); - img = frame; - } - for frame in self.flush()? { - drop(img); - img = frame; - } - if img.is_null() { - Ok(Vec::new()) - } else { - let mut out = Default::default(); - img.rgb(1, rgba, &mut out); - Ok(out) - } - } - - /// Feed some compressed data to the encoder - /// - /// The `data` slice is sent to the decoder - /// - /// It matches a call to `vpx_codec_decode`. - pub fn decode(&mut self, data: &[u8]) -> Result { - call_vpx!(vpx_codec_decode( - &mut self.ctx, - data.as_ptr(), - data.len() as _, - ptr::null_mut(), - 0, - )); - - Ok(DecodeFrames { - ctx: &mut self.ctx, - iter: ptr::null(), - }) - } - - /// Notify the decoder to return any pending frame - pub fn flush(&mut self) -> Result { - call_vpx!(vpx_codec_decode( - &mut self.ctx, - ptr::null(), - 0, - ptr::null_mut(), - 0 - )); - Ok(DecodeFrames { - ctx: &mut self.ctx, - iter: ptr::null(), - }) - } -} - -impl Drop for VpxDecoder { - fn drop(&mut self) { - unsafe { - let result = vpx_codec_destroy(&mut self.ctx); - if result != VPX_CODEC_OK { - panic!("failed to destroy vpx codec"); - } - } - } -} - -pub struct DecodeFrames<'a> { - ctx: &'a mut vpx_codec_ctx_t, - iter: vpx_codec_iter_t, -} - -impl<'a> Iterator for DecodeFrames<'a> { - type Item = Image; - fn next(&mut self) -> Option { - let img = unsafe { vpx_codec_get_frame(self.ctx, &mut self.iter) }; - if img.is_null() { - return None; - } else { - return Some(Image(img)); - } - } -} - -// https://chromium.googlesource.com/webm/libvpx/+/bali/vpx/src/vpx_image.c -pub struct Image(*mut vpx_image_t); -impl Image { - #[inline] - pub fn new() -> Self { - Self(std::ptr::null_mut()) - } - - #[inline] - pub fn is_null(&self) -> bool { - self.0.is_null() - } - - #[inline] - pub fn width(&self) -> usize { - self.inner().d_w as _ - } - - #[inline] - pub fn height(&self) -> usize { - self.inner().d_h as _ - } - - #[inline] - pub fn format(&self) -> vpx_img_fmt_t { - // VPX_IMG_FMT_I420 - self.inner().fmt - } - - #[inline] - pub fn inner(&self) -> &vpx_image_t { - unsafe { &*self.0 } - } - - #[inline] - pub fn stride(&self, iplane: usize) -> i32 { - self.inner().stride[iplane] - } - - pub fn rgb(&self, stride_align: usize, rgba: bool, dst: &mut Vec) { - let h = self.height(); - let mut w = self.width(); - let bps = if rgba { 4 } else { 3 }; - w = (w + stride_align - 1) & !(stride_align - 1); - dst.resize(h * w * bps, 0); - let img = self.inner(); - unsafe { - if rgba { - super::I420ToARGB( - img.planes[0], - img.stride[0], - img.planes[1], - img.stride[1], - img.planes[2], - img.stride[2], - dst.as_mut_ptr(), - (w * bps) as _, - self.width() as _, - self.height() as _, - ); - } else { - super::I420ToRAW( - img.planes[0], - img.stride[0], - img.planes[1], - img.stride[1], - img.planes[2], - img.stride[2], - dst.as_mut_ptr(), - (w * bps) as _, - self.width() as _, - self.height() as _, - ); - } - } - } - - #[inline] - pub fn data(&self) -> (&[u8], &[u8], &[u8]) { - unsafe { - let img = self.inner(); - let h = (img.d_h as usize + 1) & !1; - let n = img.stride[0] as usize * h; - let y = slice::from_raw_parts(img.planes[0], n); - let n = img.stride[1] as usize * (h >> 1); - let u = slice::from_raw_parts(img.planes[1], n); - let v = slice::from_raw_parts(img.planes[2], n); - (y, u, v) - } - } -} - -impl Drop for Image { - fn drop(&mut self) { - if !self.0.is_null() { - unsafe { vpx_img_free(self.0) }; - } - } -} - -unsafe impl Send for vpx_codec_ctx_t {} +// https://github.com/astraw/vpx-encode +// https://github.com/astraw/env-libvpx-sys +// https://github.com/rust-av/vpx-rs/blob/master/src/decoder.rs + +use hbb_common::anyhow::{anyhow, Context}; +use hbb_common::message_proto::{Message, VP9s, VideoFrame, VP9}; +use hbb_common::ResultType; + +use crate::coder::EncoderApi; +use crate::STRIDE_ALIGN; + +use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *}; +use std::os::raw::{c_int, c_uint}; +use std::{ptr, slice}; + +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum VpxVideoCodecId { + VP8, + VP9, +} + +impl Default for VpxVideoCodecId { + fn default() -> VpxVideoCodecId { + VpxVideoCodecId::VP9 + } +} + +pub struct VpxEncoder { + ctx: vpx_codec_ctx_t, + width: usize, + height: usize, +} + +pub struct VpxDecoder { + ctx: vpx_codec_ctx_t, +} + +#[derive(Debug)] +pub enum Error { + FailedCall(String), + BadPtr(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::result::Result<(), std::fmt::Error> { + write!(f, "{:?}", self) + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; + +macro_rules! call_vpx { + ($x:expr) => {{ + let result = unsafe { $x }; // original expression + let result_int = unsafe { std::mem::transmute::<_, i32>(result) }; + if result_int != 0 { + return Err(Error::FailedCall(format!( + "errcode={} {}:{}:{}:{}", + result_int, + module_path!(), + file!(), + line!(), + column!() + )) + .into()); + } + result + }}; +} + +macro_rules! call_vpx_ptr { + ($x:expr) => {{ + let result = unsafe { $x }; // original expression + let result_int = unsafe { std::mem::transmute::<_, isize>(result) }; + if result_int == 0 { + return Err(Error::BadPtr(format!( + "errcode={} {}:{}:{}:{}", + result_int, + module_path!(), + file!(), + line!(), + column!() + )) + .into()); + } + result + }}; +} + +impl EncoderApi for VpxEncoder { + fn new(cfg: crate::coder::EncoderCfg) -> ResultType + where + Self: Sized, + { + match cfg { + crate::coder::EncoderCfg::VPX(config) => { + let i; + if cfg!(feature = "VP8") { + i = match config.codec { + VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_cx()), + VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_cx()), + }; + } else { + i = call_vpx_ptr!(vpx_codec_vp9_cx()); + } + let mut c = unsafe { std::mem::MaybeUninit::zeroed().assume_init() }; + call_vpx!(vpx_codec_enc_config_default(i, &mut c, 0)); + + // https://www.webmproject.org/docs/encoder-parameters/ + // default: c.rc_min_quantizer = 0, c.rc_max_quantizer = 63 + // try rc_resize_allowed later + + c.g_w = config.width; + c.g_h = config.height; + c.g_timebase.num = config.timebase[0]; + c.g_timebase.den = config.timebase[1]; + c.rc_target_bitrate = config.bitrate; + c.rc_undershoot_pct = 95; + c.rc_dropframe_thresh = 25; + if config.rc_min_quantizer > 0 { + c.rc_min_quantizer = config.rc_min_quantizer; + } + if config.rc_max_quantizer > 0 { + c.rc_max_quantizer = config.rc_max_quantizer; + } + let mut speed = config.speed; + if speed <= 0 { + speed = 6; + } + + c.g_threads = if config.num_threads == 0 { + num_cpus::get() as _ + } else { + config.num_threads + }; + c.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT; + // https://developers.google.com/media/vp9/bitrate-modes/ + // Constant Bitrate mode (CBR) is recommended for live streaming with VP9. + c.rc_end_usage = vpx_rc_mode::VPX_CBR; + // c.kf_min_dist = 0; + // c.kf_max_dist = 999999; + c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot + + /* + VPX encoder支持two-pass encode，这是为了rate control的。 + 对于两遍编码，就是需要整个编码过程做两次，第一次会得到一些新的控制参数来进行第二遍的编码， + 这样可以在相同的bitrate下得到最好的PSNR + */ + + let mut ctx = Default::default(); + call_vpx!(vpx_codec_enc_init_ver( + &mut ctx, + i, + &c, + 0, + VPX_ENCODER_ABI_VERSION as _ + )); + + if config.codec == VpxVideoCodecId::VP9 { + // set encoder internal speed settings + // in ffmpeg, it is --speed option + /* + set to 0 or a positive value 1-16, the codec will try to adapt its + complexity depending on the time it spends encoding. Increasing this + number will make the speed go up and the quality go down. + Negative values mean strict enforcement of this + while positive values are adaptive + */ + /* https://developers.google.com/media/vp9/live-encoding + Speed 5 to 8 should be used for live / real-time encoding. + Lower numbers (5 or 6) are higher quality but require more CPU power. + Higher numbers (7 or 8) will be lower quality but more manageable for lower latency + use cases and also for lower CPU power devices such as mobile. + */ + call_vpx!(vpx_codec_control_(&mut ctx, VP8E_SET_CPUUSED as _, speed,)); + // set row level multi-threading + /* + as some people in comments and below have already commented, + more recent versions of libvpx support -row-mt 1 to enable tile row + multi-threading. This can increase the number of tiles by up to 4x in VP9 + (since the max number of tile rows is 4, regardless of video height). + To enable this, use -tile-rows N where N is the number of tile rows in + log2 units (so -tile-rows 1 means 2 tile rows and -tile-rows 2 means 4 tile + rows). The total number of active threads will then be equal to + $tile_rows * $tile_columns + */ + call_vpx!(vpx_codec_control_( + &mut ctx, + VP9E_SET_ROW_MT as _, + 1 as c_int + )); + + call_vpx!(vpx_codec_control_( + &mut ctx, + VP9E_SET_TILE_COLUMNS as _, + 4 as c_int + )); + } + + Ok(Self { + ctx, + width: config.width as _, + height: config.height as _, + }) + } + _ => Err(anyhow!("encoder type mismatch")), + } + } + + fn encode_to_message(&mut self, frame: &[u8], ms: i64) -> ResultType { + let mut frames = Vec::new(); + for ref frame in self + .encode(ms, frame, STRIDE_ALIGN) + .with_context(|| "Failed to encode")? + { + frames.push(VpxEncoder::create_frame(frame)); + } + for ref frame in self.flush().with_context(|| "Failed to flush")? { + frames.push(VpxEncoder::create_frame(frame)); + } + + // to-do: flush periodically, e.g. 1 second + if frames.len() > 0 { + Ok(VpxEncoder::create_msg(frames)) + } else { + Err(anyhow!("no valid frame")) + } + } + + fn use_yuv(&self) -> bool { + true + } +} + +impl VpxEncoder { + pub fn encode(&mut self, pts: i64, data: &[u8], stride_align: usize) -> Result { + assert!(2 * data.len() >= 3 * self.width * self.height); + + let mut image = Default::default(); + call_vpx_ptr!(vpx_img_wrap( + &mut image, + vpx_img_fmt::VPX_IMG_FMT_I420, + self.width as _, + self.height as _, + stride_align as _, + data.as_ptr() as _, + )); + + call_vpx!(vpx_codec_encode( + &mut self.ctx, + &image, + pts as _, + 1, // Duration + 0, // Flags + VPX_DL_REALTIME as _, + )); + + Ok(EncodeFrames { + ctx: &mut self.ctx, + iter: ptr::null(), + }) + } + + /// Notify the encoder to return any pending packets + pub fn flush(&mut self) -> Result { + call_vpx!(vpx_codec_encode( + &mut self.ctx, + ptr::null(), + -1, // PTS + 1, // Duration + 0, // Flags + VPX_DL_REALTIME as _, + )); + + Ok(EncodeFrames { + ctx: &mut self.ctx, + iter: ptr::null(), + }) + } + + #[inline] + fn create_msg(vp9s: Vec) -> Message { + let mut msg_out = Message::new(); + let mut vf = VideoFrame::new(); + vf.set_vp9s(VP9s { + frames: vp9s.into(), + ..Default::default() + }); + msg_out.set_video_frame(vf); + msg_out + } + + #[inline] + fn create_frame(frame: &EncodeFrame) -> VP9 { + VP9 { + data: frame.data.to_vec(), + key: frame.key, + pts: frame.pts, + ..Default::default() + } + } +} + +impl Drop for VpxEncoder { + fn drop(&mut self) { + unsafe { + let result = vpx_codec_destroy(&mut self.ctx); + if result != VPX_CODEC_OK { + panic!("failed to destroy vpx codec"); + } + } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct EncodeFrame<'a> { + /// Compressed data. + pub data: &'a [u8], + /// Whether the frame is a keyframe. + pub key: bool, + /// Presentation timestamp (in timebase units). + pub pts: i64, +} + +#[derive(Clone, Copy, Debug)] +pub struct VpxEncoderConfig { + /// The width (in pixels). + pub width: c_uint, + /// The height (in pixels). + pub height: c_uint, + /// The timebase numerator and denominator (in seconds). + pub timebase: [c_int; 2], + /// The target bitrate (in kilobits per second). + pub bitrate: c_uint, + /// The codec + pub codec: VpxVideoCodecId, + pub rc_min_quantizer: u32, + pub rc_max_quantizer: u32, + pub speed: i32, + pub num_threads: u32, +} + +#[derive(Clone, Copy, Debug)] +pub struct VpxDecoderConfig { + pub codec: VpxVideoCodecId, + pub num_threads: u32, +} + +pub struct EncodeFrames<'a> { + ctx: &'a mut vpx_codec_ctx_t, + iter: vpx_codec_iter_t, +} + +impl<'a> Iterator for EncodeFrames<'a> { + type Item = EncodeFrame<'a>; + fn next(&mut self) -> Option { + loop { + unsafe { + let pkt = vpx_codec_get_cx_data(self.ctx, &mut self.iter); + if pkt.is_null() { + return None; + } else if (*pkt).kind == vpx_codec_cx_pkt_kind::VPX_CODEC_CX_FRAME_PKT { + let f = &(*pkt).data.frame; + return Some(Self::Item { + data: slice::from_raw_parts(f.buf as _, f.sz as _), + key: (f.flags & VPX_FRAME_IS_KEY) != 0, + pts: f.pts, + }); + } else { + // Ignore the packet. + } + } + } + } +} + +impl VpxDecoder { + /// Create a new decoder + /// + /// # Errors + /// + /// The function may fail if the underlying libvpx does not provide + /// the VP9 decoder. + pub fn new(config: VpxDecoderConfig) -> Result { + // This is sound because `vpx_codec_ctx` is a repr(C) struct without any field that can + // cause UB if uninitialized. + let i; + if cfg!(feature = "VP8") { + i = match config.codec { + VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_dx()), + VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_dx()), + }; + } else { + i = call_vpx_ptr!(vpx_codec_vp9_dx()); + } + let mut ctx = Default::default(); + let cfg = vpx_codec_dec_cfg_t { + threads: if config.num_threads == 0 { + num_cpus::get() as _ + } else { + config.num_threads + }, + w: 0, + h: 0, + }; + /* + unsafe { + println!("{}", vpx_codec_get_caps(i)); + } + */ + call_vpx!(vpx_codec_dec_init_ver( + &mut ctx, + i, + &cfg, + 0, + VPX_DECODER_ABI_VERSION as _, + )); + Ok(Self { ctx }) + } + + pub fn decode2rgb(&mut self, data: &[u8], rgba: bool) -> Result> { + let mut img = Image::new(); + for frame in self.decode(data)? { + drop(img); + img = frame; + } + for frame in self.flush()? { + drop(img); + img = frame; + } + if img.is_null() { + Ok(Vec::new()) + } else { + let mut out = Default::default(); + img.rgb(1, rgba, &mut out); + Ok(out) + } + } + + /// Feed some compressed data to the encoder + /// + /// The `data` slice is sent to the decoder + /// + /// It matches a call to `vpx_codec_decode`. + pub fn decode(&mut self, data: &[u8]) -> Result { + call_vpx!(vpx_codec_decode( + &mut self.ctx, + data.as_ptr(), + data.len() as _, + ptr::null_mut(), + 0, + )); + + Ok(DecodeFrames { + ctx: &mut self.ctx, + iter: ptr::null(), + }) + } + + /// Notify the decoder to return any pending frame + pub fn flush(&mut self) -> Result { + call_vpx!(vpx_codec_decode( + &mut self.ctx, + ptr::null(), + 0, + ptr::null_mut(), + 0 + )); + Ok(DecodeFrames { + ctx: &mut self.ctx, + iter: ptr::null(), + }) + } +} + +impl Drop for VpxDecoder { + fn drop(&mut self) { + unsafe { + let result = vpx_codec_destroy(&mut self.ctx); + if result != VPX_CODEC_OK { + panic!("failed to destroy vpx codec"); + } + } + } +} + +pub struct DecodeFrames<'a> { + ctx: &'a mut vpx_codec_ctx_t, + iter: vpx_codec_iter_t, +} + +impl<'a> Iterator for DecodeFrames<'a> { + type Item = Image; + fn next(&mut self) -> Option { + let img = unsafe { vpx_codec_get_frame(self.ctx, &mut self.iter) }; + if img.is_null() { + return None; + } else { + return Some(Image(img)); + } + } +} + +// https://chromium.googlesource.com/webm/libvpx/+/bali/vpx/src/vpx_image.c +pub struct Image(*mut vpx_image_t); +impl Image { + #[inline] + pub fn new() -> Self { + Self(std::ptr::null_mut()) + } + + #[inline] + pub fn is_null(&self) -> bool { + self.0.is_null() + } + + #[inline] + pub fn width(&self) -> usize { + self.inner().d_w as _ + } + + #[inline] + pub fn height(&self) -> usize { + self.inner().d_h as _ + } + + #[inline] + pub fn format(&self) -> vpx_img_fmt_t { + // VPX_IMG_FMT_I420 + self.inner().fmt + } + + #[inline] + pub fn inner(&self) -> &vpx_image_t { + unsafe { &*self.0 } + } + + #[inline] + pub fn stride(&self, iplane: usize) -> i32 { + self.inner().stride[iplane] + } + + pub fn rgb(&self, stride_align: usize, rgba: bool, dst: &mut Vec) { + let h = self.height(); + let mut w = self.width(); + let bps = if rgba { 4 } else { 3 }; + w = (w + stride_align - 1) & !(stride_align - 1); + dst.resize(h * w * bps, 0); + let img = self.inner(); + unsafe { + if rgba { + super::I420ToARGB( + img.planes[0], + img.stride[0], + img.planes[1], + img.stride[1], + img.planes[2], + img.stride[2], + dst.as_mut_ptr(), + (w * bps) as _, + self.width() as _, + self.height() as _, + ); + } else { + super::I420ToRAW( + img.planes[0], + img.stride[0], + img.planes[1], + img.stride[1], + img.planes[2], + img.stride[2], + dst.as_mut_ptr(), + (w * bps) as _, + self.width() as _, + self.height() as _, + ); + } + } + } + + #[inline] + pub fn data(&self) -> (&[u8], &[u8], &[u8]) { + unsafe { + let img = self.inner(); + let h = (img.d_h as usize + 1) & !1; + let n = img.stride[0] as usize * h; + let y = slice::from_raw_parts(img.planes[0], n); + let n = img.stride[1] as usize * (h >> 1); + let u = slice::from_raw_parts(img.planes[1], n); + let v = slice::from_raw_parts(img.planes[2], n); + (y, u, v) + } + } +} + +impl Drop for Image { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { vpx_img_free(self.0) }; + } + } +} + +unsafe impl Send for vpx_codec_ctx_t {} diff --git a/src/server/video_service.rs b/src/server/video_service.rs index 225df306a..40ae41bdb 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -27,8 +27,8 @@ use hbb_common::tokio::{ }, }; use scrap::{ - codec::{VpxEncoderConfig, VpxVideoCodecId}, coder::{Encoder, EncoderCfg, HwEncoderConfig}, + vpxcodec::{VpxEncoderConfig, VpxVideoCodecId}, Capturer, Display, }; use std::{ From 399ddc8bef060eee7335e7bfd512e7373099b882 Mon Sep 17 00:00:00 2001 From: 21pages Date: Sun, 29 May 2022 19:22:09 +0800 Subject: [PATCH 03/12] scrap: rename coder.rs to codec.rs Signed-off-by: 21pages --- libs/scrap/examples/record-screen.rs | 2 +- libs/scrap/src/common/{coder.rs => codec.rs} | 0 libs/scrap/src/common/hwcodec.rs | 2 +- libs/scrap/src/common/mod.rs | 4 ++-- libs/scrap/src/common/vpxcodec.rs | 6 +++--- src/client.rs | 2 +- src/server/connection.rs | 6 +++--- src/server/video_service.rs | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) rename libs/scrap/src/common/{coder.rs => codec.rs} (100%) diff --git a/libs/scrap/examples/record-screen.rs b/libs/scrap/examples/record-screen.rs index 8be7dc132..ac9dc3ff9 100644 --- a/libs/scrap/examples/record-screen.rs +++ b/libs/scrap/examples/record-screen.rs @@ -13,7 +13,7 @@ use std::time::{Duration, Instant}; use std::{io, thread}; use docopt::Docopt; -use scrap::coder::{EncoderApi, EncoderCfg}; +use scrap::codec::{EncoderApi, EncoderCfg}; use webm::mux; use webm::mux::Track; diff --git a/libs/scrap/src/common/coder.rs b/libs/scrap/src/common/codec.rs similarity index 100% rename from libs/scrap/src/common/coder.rs rename to libs/scrap/src/common/codec.rs diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index f2867c592..1ca89a852 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -1,5 +1,5 @@ use crate::{ - coder::{EncoderApi, EncoderCfg}, + codec::{EncoderApi, EncoderCfg}, hw, HW_STRIDE_ALIGN, }; use hbb_common::{ diff --git a/libs/scrap/src/common/mod.rs b/libs/scrap/src/common/mod.rs index 993f20628..a95069686 100644 --- a/libs/scrap/src/common/mod.rs +++ b/libs/scrap/src/common/mod.rs @@ -27,7 +27,7 @@ cfg_if! { } } -pub mod coder; +pub mod codec; mod convert; #[cfg(feature = "hwcodec")] pub mod hwcodec; @@ -47,4 +47,4 @@ pub fn would_block_if_equal(old: &mut Vec, b: &[u8]) -> std::io::Result<() old.resize(b.len(), 0); old.copy_from_slice(b); Ok(()) -} \ No newline at end of file +} diff --git a/libs/scrap/src/common/vpxcodec.rs b/libs/scrap/src/common/vpxcodec.rs index 7301dc705..4067996d0 100644 --- a/libs/scrap/src/common/vpxcodec.rs +++ b/libs/scrap/src/common/vpxcodec.rs @@ -6,7 +6,7 @@ use hbb_common::anyhow::{anyhow, Context}; use hbb_common::message_proto::{Message, VP9s, VideoFrame, VP9}; use hbb_common::ResultType; -use crate::coder::EncoderApi; +use crate::codec::EncoderApi; use crate::STRIDE_ALIGN; use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *}; @@ -90,12 +90,12 @@ macro_rules! call_vpx_ptr { } impl EncoderApi for VpxEncoder { - fn new(cfg: crate::coder::EncoderCfg) -> ResultType + fn new(cfg: crate::codec::EncoderCfg) -> ResultType where Self: Sized, { match cfg { - crate::coder::EncoderCfg::VPX(config) => { + crate::codec::EncoderCfg::VPX(config) => { let i; if cfg!(feature = "VP8") { i = match config.codec { diff --git a/src/client.rs b/src/client.rs index 915878774..2cbf007d5 100644 --- a/src/client.rs +++ b/src/client.rs @@ -13,7 +13,7 @@ use cpal::{ }; use magnum_opus::{Channels::*, Decoder as AudioDecoder}; use scrap::{ - coder::{Decoder, DecoderCfg}, + codec::{Decoder, DecoderCfg}, VpxDecoderConfig, VpxVideoCodecId, }; diff --git a/src/server/connection.rs b/src/server/connection.rs index fea7c6605..87e90d5d9 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -365,7 +365,7 @@ impl Connection { video_service::notify_video_frame_feched(id, None); super::video_service::update_test_latency(id, 0); super::video_service::update_image_quality(id, None); - scrap::coder::Encoder::update_video_encoder(id, None); + scrap::codec::Encoder::update_video_encoder(id, None); if let Err(err) = conn.try_port_forward_loop(&mut rx_from_cm).await { conn.on_close(&err.to_string(), false); } @@ -1190,9 +1190,9 @@ impl Connection { // TODO: add option if let Some(q) = o.video_codec_state.clone().take() { - scrap::coder::Encoder::update_video_encoder(self.inner.id(), Some(q)); + scrap::codec::Encoder::update_video_encoder(self.inner.id(), Some(q)); } else { - scrap::coder::Encoder::update_video_encoder( + scrap::codec::Encoder::update_video_encoder( self.inner.id(), Some(VideoCodecState { H264: false, diff --git a/src/server/video_service.rs b/src/server/video_service.rs index 40ae41bdb..005553dc4 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -27,7 +27,7 @@ use hbb_common::tokio::{ }, }; use scrap::{ - coder::{Encoder, EncoderCfg, HwEncoderConfig}, + codec::{Encoder, EncoderCfg, HwEncoderConfig}, vpxcodec::{VpxEncoderConfig, VpxVideoCodecId}, Capturer, Display, }; From 7e6c38e6d2aabe4b73f10ffe9556383ca5779a1e Mon Sep 17 00:00:00 2001 From: 21pages Date: Wed, 1 Jun 2022 18:40:28 +0800 Subject: [PATCH 04/12] scrap: add hw quality control Signed-off-by: 21pages --- Cargo.lock | 2 +- libs/scrap/src/common/codec.rs | 32 ++++++++++++++----- libs/scrap/src/common/hwcodec.rs | 54 ++++++++++++++++++++++++++++++-- src/server/connection.rs | 13 ++++---- src/server/video_service.rs | 16 +++++++++- 5 files changed, 97 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 35df55da9..8759b1716 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2236,7 +2236,7 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hwcodec" version = "0.1.0" -source = "git+https://github.com/21pages/hwcodec#373d55d38c23cc8a9ef9961b3b2979d5fc9d1bc4" +source = "git+https://github.com/21pages/hwcodec#1f8e03c433c1ec73067b4c85026d99f2e9fcf9bf" dependencies = [ "bindgen", "cc", diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index af2412c0f..71ae1e9a0 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -11,7 +11,7 @@ use crate::vpxcodec::*; use hbb_common::{ anyhow::anyhow, - message_proto::{video_frame, Message, VP9s, VideoCodecState}, + message_proto::{video_frame, ImageQuality, Message, VP9s, VideoCodecState}, ResultType, }; #[cfg(feature = "hwcodec")] @@ -28,9 +28,9 @@ lazy_static::lazy_static! { #[derive(Debug, Clone)] pub struct HwEncoderConfig { pub codec_name: String, - pub fps: i32, pub width: usize, pub height: usize, + pub quallity: ImageQuality, } pub enum EncoderCfg { @@ -78,6 +78,13 @@ pub struct Decoder { i420: Vec, } +#[derive(Debug, Clone)] +pub enum EncoderUpdate { + State(VideoCodecState), + Remove, + DisableHwIfNotExist, +} + impl Encoder { pub fn new(config: EncoderCfg) -> ResultType { match config { @@ -95,14 +102,23 @@ impl Encoder { } // TODO - pub fn update_video_encoder(id: i32, decoder: Option) { + pub fn update_video_encoder(id: i32, update: EncoderUpdate) { #[cfg(feature = "hwcodec")] { let mut states = VIDEO_CODEC_STATES.lock().unwrap(); - match decoder { - Some(decoder) => states.insert(id, decoder), - None => states.remove(&id), - }; + match update { + EncoderUpdate::State(state) => { + states.insert(id, state); + } + EncoderUpdate::Remove => { + states.remove(&id); + } + EncoderUpdate::DisableHwIfNotExist => { + if !states.contains_key(&id) { + states.insert(id, VideoCodecState::default()); + } + } + } let (encoder_h264, encoder_h265) = HwEncoder::best(); let mut enabled_h264 = encoder_h264.is_some(); let mut enabled_h265 = encoder_h265.is_some(); @@ -142,7 +158,7 @@ impl Encoder { #[cfg(not(feature = "hwcodec"))] { let _ = id; - let _ = decoder; + let _ = update; } } #[inline] diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index 1ca89a852..6ccc3eb8a 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -5,7 +5,7 @@ use crate::{ use hbb_common::{ anyhow::{anyhow, Context}, lazy_static, log, - message_proto::{H264s, H265s, Message, VideoFrame, H264, H265}, + message_proto::{H264s, H265s, ImageQuality, Message, VideoFrame, H264, H265}, ResultType, }; use hwcodec::{ @@ -13,6 +13,8 @@ use hwcodec::{ encode::{EncodeContext, EncodeFrame, Encoder}, ffmpeg::{CodecInfo, DataFormat}, AVPixelFormat, + Quality::{self, *}, + RateContorl::{self, *}, }; use std::sync::{Arc, Mutex, Once}; @@ -25,6 +27,10 @@ lazy_static::lazy_static! { } const DEFAULT_PIXFMT: AVPixelFormat = AVPixelFormat::AV_PIX_FMT_YUV420P; +const DEFAULT_TIME_BASE: [i32; 2] = [1, 30]; +const DEFAULT_GOP: i32 = 60; +const DEFAULT_HW_QUALITY: Quality = Quality_Default; +const DEFAULT_RC: RateContorl = RC_DEFAULT; pub struct HwEncoder { encoder: Encoder, @@ -40,13 +46,19 @@ impl EncoderApi for HwEncoder { { match cfg { EncoderCfg::HW(config) => { + let (bitrate, timebase, gop, quality, rc) = + HwEncoder::convert_quality(&config.codec_name, config.quallity); let ctx = EncodeContext { name: config.codec_name.clone(), - fps: config.fps as _, width: config.width as _, height: config.height as _, pixfmt: DEFAULT_PIXFMT, align: HW_STRIDE_ALIGN as _, + bitrate, + timebase, + gop, + quality, + rc, }; let format = match Encoder::format_from_name(config.codec_name.clone()) { Ok(format) => format, @@ -131,11 +143,15 @@ impl HwEncoder { pub fn best() -> (Option, Option) { let ctx = EncodeContext { name: String::from(""), - fps: 30, width: 1920, height: 1080, pixfmt: DEFAULT_PIXFMT, align: HW_STRIDE_ALIGN as _, + bitrate: 0, + timebase: DEFAULT_TIME_BASE, + gop: DEFAULT_GOP, + quality: DEFAULT_HW_QUALITY, + rc: DEFAULT_RC, }; CodecInfo::score(Encoder::avaliable_encoders(ctx)) } @@ -175,6 +191,38 @@ impl HwEncoder { Err(_) => Ok(Vec::::new()), } } + + fn convert_quality(name: &str, q: ImageQuality) -> (i32, [i32; 2], i32, Quality, RateContorl) { + // TODO + let bitrate = if name.contains("qsv") { + 1_000_000 + } else { + 2_000_000 + }; + match q { + ImageQuality::Low => ( + bitrate / 2, + DEFAULT_TIME_BASE, + DEFAULT_GOP, + DEFAULT_HW_QUALITY, + DEFAULT_RC, + ), + ImageQuality::Best => ( + bitrate * 2, + DEFAULT_TIME_BASE, + DEFAULT_GOP, + DEFAULT_HW_QUALITY, + DEFAULT_RC, + ), + ImageQuality::NotSet | ImageQuality::Balanced => ( + 0, + DEFAULT_TIME_BASE, + DEFAULT_GOP, + DEFAULT_HW_QUALITY, + DEFAULT_RC, + ), + } + } } pub struct HwDecoder { diff --git a/src/server/connection.rs b/src/server/connection.rs index 87e90d5d9..d903a2803 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -365,7 +365,7 @@ impl Connection { video_service::notify_video_frame_feched(id, None); super::video_service::update_test_latency(id, 0); super::video_service::update_image_quality(id, None); - scrap::codec::Encoder::update_video_encoder(id, None); + scrap::codec::Encoder::update_video_encoder(id, scrap::codec::EncoderUpdate::Remove); if let Err(err) = conn.try_port_forward_loop(&mut rx_from_cm).await { conn.on_close(&err.to_string(), false); } @@ -1190,15 +1190,14 @@ impl Connection { // TODO: add option if let Some(q) = o.video_codec_state.clone().take() { - scrap::codec::Encoder::update_video_encoder(self.inner.id(), Some(q)); + scrap::codec::Encoder::update_video_encoder( + self.inner.id(), + scrap::codec::EncoderUpdate::State(q), + ); } else { scrap::codec::Encoder::update_video_encoder( self.inner.id(), - Some(VideoCodecState { - H264: false, - H265: false, - ..Default::default() - }), + scrap::codec::EncoderUpdate::DisableHwIfNotExist, ); } } diff --git a/src/server/video_service.rs b/src/server/video_service.rs index 005553dc4..46716b228 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -184,9 +184,9 @@ fn run(sp: GenericService) -> ResultType<()> { let encoder_cfg = match Encoder::current_hw_encoder_name() { Some(codec_name) => EncoderCfg::HW(HwEncoderConfig { codec_name, - fps, width, height, + quallity: convert_quality_back(q), }), None => EncoderCfg::VPX(VpxEncoderConfig { width: width as _, @@ -516,6 +516,20 @@ fn convert_quality(q: i32) -> i32 { } } +fn convert_quality_back(q: i32) -> ImageQuality { + let q = q >> 8; + if q == 100 * 2 / 3 { + ImageQuality::Balanced + } else if q == 100 / 2 { + ImageQuality::Low + } else if q == 100 { + ImageQuality::Best + } else { + log::error!("Error convert quality:{}", q); + ImageQuality::Balanced + } +} + pub fn update_image_quality(id: i32, q: Option) { match q { Some(q) => { From 327bdb741c433ca66e525de8c6a023d2be255bad Mon Sep 17 00:00:00 2001 From: 21pages Date: Thu, 2 Jun 2022 07:21:21 +0800 Subject: [PATCH 05/12] scrap: fix update_video_encoder Signed-off-by: 21pages --- libs/scrap/src/common/codec.rs | 52 +++++++++++++++++++--------------- src/server/connection.rs | 29 ++++++++++--------- 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index 71ae1e9a0..a0961d0e6 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -11,12 +11,13 @@ use crate::vpxcodec::*; use hbb_common::{ anyhow::anyhow, + log, message_proto::{video_frame, ImageQuality, Message, VP9s, VideoCodecState}, ResultType, }; #[cfg(feature = "hwcodec")] use hbb_common::{ - lazy_static, log, + lazy_static, message_proto::{H264s, H265s}, }; @@ -33,6 +34,7 @@ pub struct HwEncoderConfig { pub quallity: ImageQuality, } +#[derive(Debug, Clone)] pub enum EncoderCfg { VPX(VpxEncoderConfig), HW(HwEncoderConfig), @@ -87,6 +89,7 @@ pub enum EncoderUpdate { impl Encoder { pub fn new(config: EncoderCfg) -> ResultType { + log::info!("new encoder:{:?}", config); match config { EncoderCfg::VPX(_) => Ok(Encoder { codec: Box::new(VpxEncoder::new(config)?), @@ -103,6 +106,7 @@ impl Encoder { // TODO pub fn update_video_encoder(id: i32, update: EncoderUpdate) { + log::info!("update video encoder:{:?}", update); #[cfg(feature = "hwcodec")] { let mut states = VIDEO_CODEC_STATES.lock().unwrap(); @@ -119,30 +123,30 @@ impl Encoder { } } } - let (encoder_h264, encoder_h265) = HwEncoder::best(); - let mut enabled_h264 = encoder_h264.is_some(); - let mut enabled_h265 = encoder_h265.is_some(); - let mut score_vpx = 90; - let mut score_h264 = encoder_h264.as_ref().map_or(0, |c| c.score); - let mut score_h265 = encoder_h265.as_ref().map_or(0, |c| c.score); - - for state in states.iter() { - enabled_h264 = enabled_h264 && state.1.H264; - enabled_h265 = enabled_h265 && state.1.H265; - score_vpx += state.1.ScoreVpx; - score_h264 += state.1.ScoreH264; - score_h265 += state.1.ScoreH265; - } - let current_encoder_name = HwEncoder::current_name(); - if enabled_h265 && score_h265 >= score_vpx && score_h265 >= score_h264 { - *current_encoder_name.lock().unwrap() = Some(encoder_h265.unwrap().name); - } else if enabled_h264 && score_h264 >= score_vpx && score_h264 >= score_h265 { - *current_encoder_name.lock().unwrap() = Some(encoder_h264.unwrap().name); - } else { - *current_encoder_name.lock().unwrap() = None; - } if states.len() > 0 { + let (encoder_h264, encoder_h265) = HwEncoder::best(); + let mut enabled_h264 = encoder_h264.is_some() && states.iter().any(|(_, s)| s.H264); + let mut enabled_h265 = encoder_h265.is_some() && states.iter().any(|(_, s)| s.H265); + let mut score_vpx = 90; + let mut score_h264 = encoder_h264.as_ref().map_or(0, |c| c.score); + let mut score_h265 = encoder_h265.as_ref().map_or(0, |c| c.score); + + for state in states.iter() { + enabled_h264 = enabled_h264 && state.1.H264; + enabled_h265 = enabled_h265 && state.1.H265; + score_vpx += state.1.ScoreVpx; + score_h264 += state.1.ScoreH264; + score_h265 += state.1.ScoreH265; + } + + if enabled_h265 && score_h265 >= score_vpx && score_h265 >= score_h264 { + *current_encoder_name.lock().unwrap() = Some(encoder_h265.unwrap().name); + } else if enabled_h264 && score_h264 >= score_vpx && score_h264 >= score_h265 { + *current_encoder_name.lock().unwrap() = Some(encoder_h264.unwrap().name); + } else { + *current_encoder_name.lock().unwrap() = None; + } log::info!( "connection count:{}, h264:{}, h265:{}, score: vpx({}), h264({}), h265({}), set current encoder name {:?}", states.len(), @@ -153,6 +157,8 @@ impl Encoder { score_h265, current_encoder_name.lock().unwrap() ) + } else { + *current_encoder_name.lock().unwrap() = None; } } #[cfg(not(feature = "hwcodec"))] diff --git a/src/server/connection.rs b/src/server/connection.rs index d903a2803..168d18c24 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -781,6 +781,22 @@ impl Connection { if let Some(message::Union::login_request(lr)) = msg.union { if let Some(o) = lr.option.as_ref() { self.update_option(o).await; + if let Some(q) = o.video_codec_state.clone().take() { + scrap::codec::Encoder::update_video_encoder( + self.inner.id(), + scrap::codec::EncoderUpdate::State(q), + ); + } else { + scrap::codec::Encoder::update_video_encoder( + self.inner.id(), + scrap::codec::EncoderUpdate::DisableHwIfNotExist, + ); + } + } else { + scrap::codec::Encoder::update_video_encoder( + self.inner.id(), + scrap::codec::EncoderUpdate::DisableHwIfNotExist, + ); } self.video_ack_required = lr.video_ack_required; if self.authorized { @@ -1187,19 +1203,6 @@ impl Connection { } } } - - // TODO: add option - if let Some(q) = o.video_codec_state.clone().take() { - scrap::codec::Encoder::update_video_encoder( - self.inner.id(), - scrap::codec::EncoderUpdate::State(q), - ); - } else { - scrap::codec::Encoder::update_video_encoder( - self.inner.id(), - scrap::codec::EncoderUpdate::DisableHwIfNotExist, - ); - } } fn on_close(&mut self, reason: &str, lock: bool) { From 4bb09865cf238b9127b250cf219532d82b77eab0 Mon Sep 17 00:00:00 2001 From: 21pages Date: Sun, 5 Jun 2022 15:36:39 +0800 Subject: [PATCH 06/12] scrap: update HwDecoder::instance() Signed-off-by: 21pages --- Cargo.lock | 2 +- libs/scrap/src/common/hwcodec.rs | 34 +++++++------------------------- 2 files changed, 8 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8759b1716..6e17784ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2236,7 +2236,7 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hwcodec" version = "0.1.0" -source = "git+https://github.com/21pages/hwcodec#1f8e03c433c1ec73067b4c85026d99f2e9fcf9bf" +source = "git+https://github.com/21pages/hwcodec#1cfe82f08f254e88f18cea1ad7c332f1c28dc1b9" dependencies = [ "bindgen", "cc", diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index 6ccc3eb8a..338045526 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -240,35 +240,15 @@ impl HwDecoder { static ONCE: Once = Once::new(); // TODO: different process ONCE.call_once(|| { - let avaliable = Decoder::avaliable_decoders(); - let mut decoders = vec![]; - for decoder in avaliable { - if let Ok(d) = HwDecoder::new(decoder) { - decoders.push(d); - } - } - + let (h264_info, h265_info) = CodecInfo::score(Decoder::avaliable_decoders()); let mut h264: Option = None; let mut h265: Option = None; - for decoder in decoders { - match decoder.info.format { - DataFormat::H264 => match &h264 { - Some(old) => { - if decoder.info.score > old.info.score { - h264 = Some(decoder) - } - } - None => h264 = Some(decoder), - }, - DataFormat::H265 => match &h265 { - Some(old) => { - if decoder.info.score > old.info.score { - h265 = Some(decoder) - } - } - None => h265 = Some(decoder), - }, - } + + if let Some(info) = h264_info { + h264 = HwDecoder::new(info).ok(); + } + if let Some(info) = h265_info { + h265 = HwDecoder::new(info).ok(); } if h264.is_some() { log::info!("h264 decoder:{:?}", h264.as_ref().unwrap().info); From 42546a7468986197901e4f8ea128dd95e127d886 Mon Sep 17 00:00:00 2001 From: 21pages Date: Mon, 6 Jun 2022 14:53:29 +0800 Subject: [PATCH 07/12] scrap: use the same bitrate ratio control as vpx Signed-off-by: 21pages --- Cargo.lock | 2 +- libs/scrap/src/common/codec.rs | 4 +-- libs/scrap/src/common/hwcodec.rs | 44 ++++++++++++-------------------- src/server/video_service.rs | 16 +----------- 4 files changed, 21 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e17784ce..b07495ad3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2236,7 +2236,7 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hwcodec" version = "0.1.0" -source = "git+https://github.com/21pages/hwcodec#1cfe82f08f254e88f18cea1ad7c332f1c28dc1b9" +source = "git+https://github.com/21pages/hwcodec#888af61b3e960d30ef1f2e49eb0bcf9f6f7a14ee" dependencies = [ "bindgen", "cc", diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index a0961d0e6..66e8963f3 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -12,7 +12,7 @@ use crate::vpxcodec::*; use hbb_common::{ anyhow::anyhow, log, - message_proto::{video_frame, ImageQuality, Message, VP9s, VideoCodecState}, + message_proto::{video_frame, Message, VP9s, VideoCodecState}, ResultType, }; #[cfg(feature = "hwcodec")] @@ -31,7 +31,7 @@ pub struct HwEncoderConfig { pub codec_name: String, pub width: usize, pub height: usize, - pub quallity: ImageQuality, + pub bitrate_ratio: i32, } #[derive(Debug, Clone)] diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index 338045526..6393be3eb 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -5,7 +5,7 @@ use crate::{ use hbb_common::{ anyhow::{anyhow, Context}, lazy_static, log, - message_proto::{H264s, H265s, ImageQuality, Message, VideoFrame, H264, H265}, + message_proto::{H264s, H265s, Message, VideoFrame, H264, H265}, ResultType, }; use hwcodec::{ @@ -47,7 +47,7 @@ impl EncoderApi for HwEncoder { match cfg { EncoderCfg::HW(config) => { let (bitrate, timebase, gop, quality, rc) = - HwEncoder::convert_quality(&config.codec_name, config.quallity); + HwEncoder::convert_quality(&config.codec_name, config.bitrate_ratio); let ctx = EncodeContext { name: config.codec_name.clone(), width: config.width as _, @@ -192,36 +192,26 @@ impl HwEncoder { } } - fn convert_quality(name: &str, q: ImageQuality) -> (i32, [i32; 2], i32, Quality, RateContorl) { + fn convert_quality( + name: &str, + bitrate_ratio: i32, + ) -> (i32, [i32; 2], i32, Quality, RateContorl) { // TODO - let bitrate = if name.contains("qsv") { + let mut bitrate = if name.contains("qsv") { 1_000_000 } else { 2_000_000 }; - match q { - ImageQuality::Low => ( - bitrate / 2, - DEFAULT_TIME_BASE, - DEFAULT_GOP, - DEFAULT_HW_QUALITY, - DEFAULT_RC, - ), - ImageQuality::Best => ( - bitrate * 2, - DEFAULT_TIME_BASE, - DEFAULT_GOP, - DEFAULT_HW_QUALITY, - DEFAULT_RC, - ), - ImageQuality::NotSet | ImageQuality::Balanced => ( - 0, - DEFAULT_TIME_BASE, - DEFAULT_GOP, - DEFAULT_HW_QUALITY, - DEFAULT_RC, - ), - } + if bitrate_ratio > 0 && bitrate_ratio <= 200 { + bitrate = bitrate * bitrate_ratio / 100; + }; + ( + bitrate, + DEFAULT_TIME_BASE, + DEFAULT_GOP, + DEFAULT_HW_QUALITY, + DEFAULT_RC, + ) } } diff --git a/src/server/video_service.rs b/src/server/video_service.rs index 46716b228..f3da166db 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -186,7 +186,7 @@ fn run(sp: GenericService) -> ResultType<()> { codec_name, width, height, - quallity: convert_quality_back(q), + bitrate_ratio: q >> 8, }), None => EncoderCfg::VPX(VpxEncoderConfig { width: width as _, @@ -516,20 +516,6 @@ fn convert_quality(q: i32) -> i32 { } } -fn convert_quality_back(q: i32) -> ImageQuality { - let q = q >> 8; - if q == 100 * 2 / 3 { - ImageQuality::Balanced - } else if q == 100 / 2 { - ImageQuality::Low - } else if q == 100 { - ImageQuality::Best - } else { - log::error!("Error convert quality:{}", q); - ImageQuality::Balanced - } -} - pub fn update_image_quality(id: i32, q: Option) { match q { Some(q) => { From 91012b5da53a6591cd7d3a6092368cc0c2b01edf Mon Sep 17 00:00:00 2001 From: 21pages Date: Mon, 6 Jun 2022 17:02:07 +0800 Subject: [PATCH 08/12] scrap: refactor update_video_encoder Signed-off-by: 21pages --- libs/scrap/src/common/codec.rs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index 66e8963f3..1be47dc12 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -126,18 +126,25 @@ impl Encoder { let current_encoder_name = HwEncoder::current_name(); if states.len() > 0 { let (encoder_h264, encoder_h265) = HwEncoder::best(); - let mut enabled_h264 = encoder_h264.is_some() && states.iter().any(|(_, s)| s.H264); - let mut enabled_h265 = encoder_h265.is_some() && states.iter().any(|(_, s)| s.H265); + let enabled_h264 = encoder_h264.is_some() + && states.len() > 0 + && states.iter().all(|(_, s)| s.H264); + let enabled_h265 = encoder_h265.is_some() + && states.len() > 0 + && states.iter().all(|(_, s)| s.H265); + + // score encoder let mut score_vpx = 90; let mut score_h264 = encoder_h264.as_ref().map_or(0, |c| c.score); let mut score_h265 = encoder_h265.as_ref().map_or(0, |c| c.score); - for state in states.iter() { - enabled_h264 = enabled_h264 && state.1.H264; - enabled_h265 = enabled_h265 && state.1.H265; - score_vpx += state.1.ScoreVpx; - score_h264 += state.1.ScoreH264; - score_h265 += state.1.ScoreH265; + // score decoder + score_vpx += states.iter().map(|s| s.1.ScoreVpx).sum::(); + if enabled_h264 { + score_h264 += states.iter().map(|s| s.1.ScoreH264).sum::(); + } + if enabled_h265 { + score_h265 += states.iter().map(|s| s.1.ScoreH265).sum::(); } if enabled_h265 && score_h265 >= score_vpx && score_h265 >= score_h264 { From 27091dec0e66858b0d71dd2d86cf24ed09f30d79 Mon Sep 17 00:00:00 2001 From: 21pages Date: Tue, 7 Jun 2022 10:21:02 +0800 Subject: [PATCH 09/12] scrap: remove lock on hwDecoder Signed-off-by: 21pages --- libs/scrap/src/common/codec.rs | 18 +++++------ libs/scrap/src/common/hwcodec.rs | 54 +++++++++++++++----------------- 2 files changed, 34 insertions(+), 38 deletions(-) diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index 1be47dc12..e6c925b28 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -75,7 +75,7 @@ impl DerefMut for Encoder { pub struct Decoder { vpx: VpxDecoder, #[cfg(feature = "hwcodec")] - hw: Arc>, + hw: HwDecoders, #[cfg(feature = "hwcodec")] i420: Vec, } @@ -191,11 +191,11 @@ impl Decoder { #[cfg(feature = "hwcodec")] { - let hw = HwDecoder::instance(); - state.H264 = hw.lock().unwrap().h264.is_some(); - state.ScoreH264 = hw.lock().unwrap().h264.as_ref().map_or(0, |d| d.info.score); - state.H265 = hw.lock().unwrap().h265.is_some(); - state.ScoreH265 = hw.lock().unwrap().h265.as_ref().map_or(0, |d| d.info.score); + let (h264, h265) = super::hwcodec::HwDecoder::best(); + state.H264 = h264.is_some(); + state.ScoreH264 = h264.map_or(0, |c| c.score); + state.H265 = h265.is_some(); + state.ScoreH265 = h265.map_or(0, |c| c.score); } state @@ -206,7 +206,7 @@ impl Decoder { Decoder { vpx, #[cfg(feature = "hwcodec")] - hw: HwDecoder::instance(), + hw: HwDecoder::new_decoders(), #[cfg(feature = "hwcodec")] i420: vec![], } @@ -223,7 +223,7 @@ impl Decoder { } #[cfg(feature = "hwcodec")] video_frame::Union::h264s(h264s) => { - if let Some(decoder) = &mut self.hw.lock().unwrap().h264 { + if let Some(decoder) = &mut self.hw.h264 { Decoder::handle_h264s_video_frame(decoder, h264s, rgb, &mut self.i420) } else { Err(anyhow!("don't support h264!")) @@ -231,7 +231,7 @@ impl Decoder { } #[cfg(feature = "hwcodec")] video_frame::Union::h265s(h265s) => { - if let Some(decoder) = &mut self.hw.lock().unwrap().h265 { + if let Some(decoder) = &mut self.hw.h265 { Decoder::handle_h265s_video_frame(decoder, h265s, rgb, &mut self.i420) } else { Err(anyhow!("don't support h265!")) diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index 6393be3eb..95dcd0353 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -16,14 +16,10 @@ use hwcodec::{ Quality::{self, *}, RateContorl::{self, *}, }; -use std::sync::{Arc, Mutex, Once}; +use std::sync::{Arc, Mutex}; lazy_static::lazy_static! { static ref HW_ENCODER_NAME: Arc>> = Default::default(); - static ref HW_DECODER_INSTANCE: Arc> = Arc::new(Mutex::new(HwDecoderInstance { - h264: None, - h265: None, - })); } const DEFAULT_PIXFMT: AVPixelFormat = AVPixelFormat::AV_PIX_FMT_YUV420P; @@ -220,36 +216,36 @@ pub struct HwDecoder { pub info: CodecInfo, } -pub struct HwDecoderInstance { +pub struct HwDecoders { pub h264: Option, pub h265: Option, } impl HwDecoder { - pub fn instance() -> Arc> { - static ONCE: Once = Once::new(); - // TODO: different process - ONCE.call_once(|| { - let (h264_info, h265_info) = CodecInfo::score(Decoder::avaliable_decoders()); - let mut h264: Option = None; - let mut h265: Option = None; + /// H264, H265 decoder info with the highest score. + /// Because available_decoders is singleton, it returns same result in the same process. + pub fn best() -> (Option, Option) { + CodecInfo::score(Decoder::avaliable_decoders()) + } - if let Some(info) = h264_info { - h264 = HwDecoder::new(info).ok(); - } - if let Some(info) = h265_info { - h265 = HwDecoder::new(info).ok(); - } - if h264.is_some() { - log::info!("h264 decoder:{:?}", h264.as_ref().unwrap().info); - } - if h265.is_some() { - log::info!("h265 decoder:{:?}", h265.as_ref().unwrap().info); - } - HW_DECODER_INSTANCE.lock().unwrap().h264 = h264; - HW_DECODER_INSTANCE.lock().unwrap().h265 = h265; - }); - HW_DECODER_INSTANCE.clone() + pub fn new_decoders() -> HwDecoders { + let (h264_info, h265_info) = HwDecoder::best(); + let mut h264: Option = None; + let mut h265: Option = None; + + if let Some(info) = h264_info { + h264 = HwDecoder::new(info).ok(); + } + if let Some(info) = h265_info { + h265 = HwDecoder::new(info).ok(); + } + if h264.is_some() { + log::info!("h264 decoder:{:?}", h264.as_ref().unwrap().info); + } + if h265.is_some() { + log::info!("h265 decoder:{:?}", h265.as_ref().unwrap().info); + } + HwDecoders { h264, h265 } } pub fn new(info: CodecInfo) -> ResultType { From 2a91fb842d9e5d4b18074472387c5efe0e076d07 Mon Sep 17 00:00:00 2001 From: 21pages Date: Tue, 7 Jun 2022 19:35:18 +0800 Subject: [PATCH 10/12] scrap: save best codec info in LocalConfig Signed-off-by: 21pages --- Cargo.lock | 5 +- libs/scrap/src/common/codec.rs | 30 +++++------ libs/scrap/src/common/hwcodec.rs | 92 ++++++++++++++++++++++++-------- 3 files changed, 89 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b07495ad3..a42715628 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2236,11 +2236,14 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hwcodec" version = "0.1.0" -source = "git+https://github.com/21pages/hwcodec#888af61b3e960d30ef1f2e49eb0bcf9f6f7a14ee" +source = "git+https://github.com/21pages/hwcodec#6bb387828c9aa69861b707b0f71472b21b5b1711" dependencies = [ "bindgen", "cc", "log", + "serde 1.0.136", + "serde_derive", + "serde_json 1.0.79", ] [[package]] diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index e6c925b28..d57b77b40 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -125,18 +125,16 @@ impl Encoder { } let current_encoder_name = HwEncoder::current_name(); if states.len() > 0 { - let (encoder_h264, encoder_h265) = HwEncoder::best(); - let enabled_h264 = encoder_h264.is_some() - && states.len() > 0 - && states.iter().all(|(_, s)| s.H264); - let enabled_h265 = encoder_h265.is_some() - && states.len() > 0 - && states.iter().all(|(_, s)| s.H265); + let best = HwEncoder::best(); + let enabled_h264 = + best.h264.is_some() && states.len() > 0 && states.iter().all(|(_, s)| s.H264); + let enabled_h265 = + best.h265.is_some() && states.len() > 0 && states.iter().all(|(_, s)| s.H265); // score encoder let mut score_vpx = 90; - let mut score_h264 = encoder_h264.as_ref().map_or(0, |c| c.score); - let mut score_h265 = encoder_h265.as_ref().map_or(0, |c| c.score); + let mut score_h264 = best.h264.as_ref().map_or(0, |c| c.score); + let mut score_h265 = best.h265.as_ref().map_or(0, |c| c.score); // score decoder score_vpx += states.iter().map(|s| s.1.ScoreVpx).sum::(); @@ -148,9 +146,9 @@ impl Encoder { } if enabled_h265 && score_h265 >= score_vpx && score_h265 >= score_h264 { - *current_encoder_name.lock().unwrap() = Some(encoder_h265.unwrap().name); + *current_encoder_name.lock().unwrap() = Some(best.h265.unwrap().name); } else if enabled_h264 && score_h264 >= score_vpx && score_h264 >= score_h265 { - *current_encoder_name.lock().unwrap() = Some(encoder_h264.unwrap().name); + *current_encoder_name.lock().unwrap() = Some(best.h264.unwrap().name); } else { *current_encoder_name.lock().unwrap() = None; } @@ -191,11 +189,11 @@ impl Decoder { #[cfg(feature = "hwcodec")] { - let (h264, h265) = super::hwcodec::HwDecoder::best(); - state.H264 = h264.is_some(); - state.ScoreH264 = h264.map_or(0, |c| c.score); - state.H265 = h265.is_some(); - state.ScoreH265 = h265.map_or(0, |c| c.score); + let best = super::hwcodec::HwDecoder::best(false); + state.H264 = best.h264.is_some(); + state.ScoreH264 = best.h264.map_or(0, |c| c.score); + state.H265 = best.h265.is_some(); + state.ScoreH265 = best.h265.map_or(0, |c| c.score); } state diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index 95dcd0353..18c153587 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -4,6 +4,7 @@ use crate::{ }; use hbb_common::{ anyhow::{anyhow, Context}, + config::LocalConfig, lazy_static, log, message_proto::{H264s, H265s, Message, VideoFrame, H264, H265}, ResultType, @@ -11,7 +12,7 @@ use hbb_common::{ use hwcodec::{ decode::{DecodeContext, DecodeFrame, Decoder}, encode::{EncodeContext, EncodeFrame, Encoder}, - ffmpeg::{CodecInfo, DataFormat}, + ffmpeg::{CodecInfo, CodecInfos, DataFormat}, AVPixelFormat, Quality::{self, *}, RateContorl::{self, *}, @@ -136,20 +137,30 @@ impl EncoderApi for HwEncoder { } impl HwEncoder { - pub fn best() -> (Option, Option) { - let ctx = EncodeContext { - name: String::from(""), - width: 1920, - height: 1080, - pixfmt: DEFAULT_PIXFMT, - align: HW_STRIDE_ALIGN as _, - bitrate: 0, - timebase: DEFAULT_TIME_BASE, - gop: DEFAULT_GOP, - quality: DEFAULT_HW_QUALITY, - rc: DEFAULT_RC, - }; - CodecInfo::score(Encoder::avaliable_encoders(ctx)) + pub fn best() -> CodecInfos { + let key = "bestHwEncoders"; + match get_config(key) { + Ok(config) => config, + Err(_) => { + let ctx = EncodeContext { + name: String::from(""), + width: 1920, + height: 1080, + pixfmt: DEFAULT_PIXFMT, + align: HW_STRIDE_ALIGN as _, + bitrate: 0, + timebase: DEFAULT_TIME_BASE, + gop: DEFAULT_GOP, + quality: DEFAULT_HW_QUALITY, + rc: DEFAULT_RC, + }; + let encoders = CodecInfo::score(Encoder::avaliable_encoders(ctx)); + let _ = set_config(key, &encoders) + .map_err(|e| log::error!("{:?}", e)) + .ok(); + encoders + } + } } pub fn current_name() -> Arc>> { @@ -223,22 +234,43 @@ pub struct HwDecoders { impl HwDecoder { /// H264, H265 decoder info with the highest score. - /// Because available_decoders is singleton, it returns same result in the same process. - pub fn best() -> (Option, Option) { - CodecInfo::score(Decoder::avaliable_decoders()) + pub fn best(force_reset: bool) -> CodecInfos { + let key = "bestHwDecoders"; + let config = get_config(key); + if !force_reset && config.is_ok() { + config.unwrap() + } else { + let decoders = CodecInfo::score(Decoder::avaliable_decoders()); + set_config(key, &decoders) + .map_err(|e| log::error!("{:?}", e)) + .ok(); + decoders + } } pub fn new_decoders() -> HwDecoders { - let (h264_info, h265_info) = HwDecoder::best(); + let best = HwDecoder::best(false); let mut h264: Option = None; let mut h265: Option = None; + let mut fail = false; - if let Some(info) = h264_info { + if let Some(info) = best.h264 { h264 = HwDecoder::new(info).ok(); + if h264.is_none() { + fail = true; + } } - if let Some(info) = h265_info { + if let Some(info) = best.h265 { h265 = HwDecoder::new(info).ok(); + if h265.is_none() { + fail = true; + } } + if fail { + HwDecoder::best(true); + // TODO: notify encoder + } + if h264.is_some() { log::info!("h264 decoder:{:?}", h264.as_ref().unwrap().info); } @@ -302,3 +334,21 @@ impl HwDecoderImage<'_> { } } } + +fn get_config(k: &str) -> ResultType { + let v = LocalConfig::get_option(k); + match CodecInfos::deserialize(&v) { + Ok(v) => Ok(v), + Err(_) => Err(anyhow!("Failed to get config:{}", k)), + } +} + +fn set_config(k: &str, v: &CodecInfos) -> ResultType<()> { + match v.serialize() { + Ok(v) => { + LocalConfig::set_option(k.to_owned(), v); + Ok(()) + } + Err(_) => Err(anyhow!("Failed to set config:{}", k)), + } +} From feaadcfc9677f846bf82449b2f74d8fb4ba02feb Mon Sep 17 00:00:00 2001 From: 21pages Date: Thu, 9 Jun 2022 17:14:26 +0800 Subject: [PATCH 11/12] scrap: ensure video_handler's creation before client start Signed-off-by: 21pages --- libs/scrap/src/common/codec.rs | 65 ++++++++++++++++++++++---------- libs/scrap/src/common/hwcodec.rs | 48 +++++++++++------------ src/client.rs | 4 +- 3 files changed, 72 insertions(+), 45 deletions(-) diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index d57b77b40..9e9a824dd 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -23,8 +23,10 @@ use hbb_common::{ #[cfg(feature = "hwcodec")] lazy_static::lazy_static! { - static ref VIDEO_CODEC_STATES: Arc>> = Default::default(); + static ref PEER_DECODER_STATES: Arc>> = Default::default(); + static ref MY_DECODER_STATE: Arc> = Default::default(); } +const SCORE_VPX: i32 = 90; #[derive(Debug, Clone)] pub struct HwEncoderConfig { @@ -96,9 +98,15 @@ impl Encoder { }), #[cfg(feature = "hwcodec")] - EncoderCfg::HW(_) => Ok(Encoder { - codec: Box::new(HwEncoder::new(config)?), - }), + EncoderCfg::HW(_) => match HwEncoder::new(config) { + Ok(hw) => Ok(Encoder { + codec: Box::new(hw), + }), + Err(e) => { + HwEncoder::best(true); + Err(e) + } + }, #[cfg(not(feature = "hwcodec"))] _ => Err(anyhow!("unsupported encoder type")), } @@ -109,7 +117,7 @@ impl Encoder { log::info!("update video encoder:{:?}", update); #[cfg(feature = "hwcodec")] { - let mut states = VIDEO_CODEC_STATES.lock().unwrap(); + let mut states = PEER_DECODER_STATES.lock().unwrap(); match update { EncoderUpdate::State(state) => { states.insert(id, state); @@ -125,14 +133,14 @@ impl Encoder { } let current_encoder_name = HwEncoder::current_name(); if states.len() > 0 { - let best = HwEncoder::best(); + let best = HwEncoder::best(false); let enabled_h264 = best.h264.is_some() && states.len() > 0 && states.iter().all(|(_, s)| s.H264); let enabled_h265 = best.h265.is_some() && states.len() > 0 && states.iter().all(|(_, s)| s.H265); // score encoder - let mut score_vpx = 90; + let mut score_vpx = SCORE_VPX; let mut score_h264 = best.h264.as_ref().map_or(0, |c| c.score); let mut score_h265 = best.h265.as_ref().map_or(0, |c| c.score); @@ -181,33 +189,50 @@ impl Encoder { } } +#[cfg(feature = "hwcodec")] +impl Drop for Decoder { + fn drop(&mut self) { + *MY_DECODER_STATE.lock().unwrap() = VideoCodecState { + ScoreVpx: SCORE_VPX, + ..Default::default() + }; + } +} + impl Decoder { - // TODO pub fn video_codec_state() -> VideoCodecState { - let mut state = VideoCodecState::default(); - state.ScoreVpx = 90; - + // video_codec_state is mainted by creation and destruction of Decoder. + // It has been ensured to use after Decoder's creation. #[cfg(feature = "hwcodec")] - { - let best = super::hwcodec::HwDecoder::best(false); - state.H264 = best.h264.is_some(); - state.ScoreH264 = best.h264.map_or(0, |c| c.score); - state.H265 = best.h265.is_some(); - state.ScoreH265 = best.h265.map_or(0, |c| c.score); + return MY_DECODER_STATE.lock().unwrap().clone(); + #[cfg(not(feature = "hwcodec"))] + VideoCodecState { + ScoreVpx: SCORE_VPX, + ..Default::default() } - - state } pub fn new(config: DecoderCfg) -> Decoder { let vpx = VpxDecoder::new(config.vpx).unwrap(); - Decoder { + let decoder = Decoder { vpx, #[cfg(feature = "hwcodec")] hw: HwDecoder::new_decoders(), #[cfg(feature = "hwcodec")] i420: vec![], + }; + + #[cfg(feature = "hwcodec")] + { + let mut state = MY_DECODER_STATE.lock().unwrap(); + state.ScoreVpx = SCORE_VPX; + state.H264 = decoder.hw.h264.is_some(); + state.ScoreH264 = decoder.hw.h264.as_ref().map_or(0, |d| d.info.score); + state.H265 = decoder.hw.h265.is_some(); + state.ScoreH265 = decoder.hw.h265.as_ref().map_or(0, |d| d.info.score); } + + decoder } pub fn handle_video_frame( diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index 18c153587..2d7b6b138 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -137,29 +137,30 @@ impl EncoderApi for HwEncoder { } impl HwEncoder { - pub fn best() -> CodecInfos { + pub fn best(force_reset: bool) -> CodecInfos { let key = "bestHwEncoders"; - match get_config(key) { - Ok(config) => config, - Err(_) => { - let ctx = EncodeContext { - name: String::from(""), - width: 1920, - height: 1080, - pixfmt: DEFAULT_PIXFMT, - align: HW_STRIDE_ALIGN as _, - bitrate: 0, - timebase: DEFAULT_TIME_BASE, - gop: DEFAULT_GOP, - quality: DEFAULT_HW_QUALITY, - rc: DEFAULT_RC, - }; - let encoders = CodecInfo::score(Encoder::avaliable_encoders(ctx)); - let _ = set_config(key, &encoders) - .map_err(|e| log::error!("{:?}", e)) - .ok(); - encoders - } + + let config = get_config(key); + if !force_reset && config.is_ok() { + config.unwrap() + } else { + let ctx = EncodeContext { + name: String::from(""), + width: 1920, + height: 1080, + pixfmt: DEFAULT_PIXFMT, + align: HW_STRIDE_ALIGN as _, + bitrate: 0, + timebase: DEFAULT_TIME_BASE, + gop: DEFAULT_GOP, + quality: DEFAULT_HW_QUALITY, + rc: DEFAULT_RC, + }; + let encoders = CodecInfo::score(Encoder::avaliable_encoders(ctx)); + let _ = set_config(key, &encoders) + .map_err(|e| log::error!("{:?}", e)) + .ok(); + encoders } } @@ -234,7 +235,7 @@ pub struct HwDecoders { impl HwDecoder { /// H264, H265 decoder info with the highest score. - pub fn best(force_reset: bool) -> CodecInfos { + fn best(force_reset: bool) -> CodecInfos { let key = "bestHwDecoders"; let config = get_config(key); if !force_reset && config.is_ok() { @@ -268,7 +269,6 @@ impl HwDecoder { } if fail { HwDecoder::best(true); - // TODO: notify encoder } if h264.is_some() { diff --git a/src/client.rs b/src/client.rs index 2cbf007d5..c4ebbc537 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1159,9 +1159,11 @@ where let latency_controller = LatencyController::new(); let latency_controller_cl = latency_controller.clone(); + // Create video_handler out of the thread below to ensure that the handler exists before client start. + // It will take a few tenths of a second for the first time, and then tens of milliseconds. + let mut video_handler = VideoHandler::new(latency_controller); std::thread::spawn(move || { - let mut video_handler = VideoHandler::new(latency_controller); loop { if let Ok(data) = video_receiver.recv() { match data { From 42c7c5982c977181306928772568e729ee05a6c6 Mon Sep 17 00:00:00 2001 From: 21pages Date: Thu, 9 Jun 2022 19:46:41 +0800 Subject: [PATCH 12/12] scrap: check hwconfig in another process Signed-off-by: 21pages --- Cargo.lock | 2 +- Cargo.toml | 1 + libs/hbb_common/src/config.rs | 38 ++++++++++++++ libs/scrap/src/common/codec.rs | 6 +-- libs/scrap/src/common/hwcodec.rs | 88 +++++++++++++++++++++----------- src/ipc.rs | 29 ++++++++++- src/main.rs | 13 +++++ 7 files changed, 141 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a42715628..7decf58f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2236,7 +2236,7 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hwcodec" version = "0.1.0" -source = "git+https://github.com/21pages/hwcodec#6bb387828c9aa69861b707b0f71472b21b5b1711" +source = "git+https://github.com/21pages/hwcodec#3ef5b674d3721699daba1f78569eb9c706cb206c" dependencies = [ "bindgen", "cc", diff --git a/Cargo.toml b/Cargo.toml index 4b5b5dd29..7c0946754 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ cli = [] use_samplerate = ["samplerate"] use_rubato = ["rubato"] use_dasp = ["dasp"] +hwcodec = ["scrap/hwcodec"] default = ["use_dasp"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/libs/hbb_common/src/config.rs b/libs/hbb_common/src/config.rs index ce0fc509a..6bfef1f97 100644 --- a/libs/hbb_common/src/config.rs +++ b/libs/hbb_common/src/config.rs @@ -35,6 +35,7 @@ lazy_static::lazy_static! { static ref CONFIG: Arc> = Arc::new(RwLock::new(Config::load())); static ref CONFIG2: Arc> = Arc::new(RwLock::new(Config2::load())); static ref LOCAL_CONFIG: Arc> = Arc::new(RwLock::new(LocalConfig::load())); + static ref HWCODEC_CONFIG: Arc> = Arc::new(RwLock::new(HwCodecConfig::load())); pub static ref ONLINE: Arc>> = Default::default(); pub static ref PROD_RENDEZVOUS_SERVER: Arc> = Default::default(); pub static ref APP_NAME: Arc> = Arc::new(RwLock::new("RustDesk".to_owned())); @@ -871,6 +872,43 @@ impl LanPeers { } } +#[derive(Debug, Default, Serialize, Deserialize, Clone)] +pub struct HwCodecConfig { + #[serde(default)] + options: HashMap, +} + +impl HwCodecConfig { + fn load() -> HwCodecConfig { + Config::load_::("_hwcodec") + } + + fn store(&self) { + Config::store_(self, "_hwcodec"); + } + + pub fn get_option(k: &str) -> String { + if let Some(v) = HWCODEC_CONFIG.read().unwrap().options.get(k) { + v.clone() + } else { + "".to_owned() + } + } + + pub fn set_option(k: String, v: String) { + let mut config = HWCODEC_CONFIG.write().unwrap(); + let v2 = if v.is_empty() { None } else { Some(&v) }; + if v2 != config.options.get(&k) { + if v2.is_none() { + config.options.remove(&k); + } else { + config.options.insert(k, v); + } + config.store(); + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index 9e9a824dd..bfc93fabc 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -103,7 +103,7 @@ impl Encoder { codec: Box::new(hw), }), Err(e) => { - HwEncoder::best(true); + HwEncoder::best(true, true); Err(e) } }, @@ -114,7 +114,7 @@ impl Encoder { // TODO pub fn update_video_encoder(id: i32, update: EncoderUpdate) { - log::info!("update video encoder:{:?}", update); + log::info!("encoder update: {:?}", update); #[cfg(feature = "hwcodec")] { let mut states = PEER_DECODER_STATES.lock().unwrap(); @@ -133,7 +133,7 @@ impl Encoder { } let current_encoder_name = HwEncoder::current_name(); if states.len() > 0 { - let best = HwEncoder::best(false); + let (best, _) = HwEncoder::best(false, true); let enabled_h264 = best.h264.is_some() && states.len() > 0 && states.iter().all(|(_, s)| s.H264); let enabled_h265 = diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index 2d7b6b138..ec4e7082c 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -4,7 +4,7 @@ use crate::{ }; use hbb_common::{ anyhow::{anyhow, Context}, - config::LocalConfig, + config::HwCodecConfig, lazy_static, log, message_proto::{H264s, H265s, Message, VideoFrame, H264, H265}, ResultType, @@ -17,12 +17,18 @@ use hwcodec::{ Quality::{self, *}, RateContorl::{self, *}, }; -use std::sync::{Arc, Mutex}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; lazy_static::lazy_static! { static ref HW_ENCODER_NAME: Arc>> = Default::default(); } +const CFG_KEY_ENCODER: &str = "bestHwEncoders"; +const CFG_KEY_DECODER: &str = "bestHwDecoders"; + const DEFAULT_PIXFMT: AVPixelFormat = AVPixelFormat::AV_PIX_FMT_YUV420P; const DEFAULT_TIME_BASE: [i32; 2] = [1, 30]; const DEFAULT_GOP: i32 = 60; @@ -137,12 +143,19 @@ impl EncoderApi for HwEncoder { } impl HwEncoder { - pub fn best(force_reset: bool) -> CodecInfos { - let key = "bestHwEncoders"; - - let config = get_config(key); + /// Get best encoders. + /// + /// # Parameter + /// `force_reset`: force to refresh config. + /// `write`: write to config file. + /// + /// # Return + /// `CodecInfos`: infos. + /// `bool`: whether the config is refreshed. + pub fn best(force_reset: bool, write: bool) -> (CodecInfos, bool) { + let config = get_config(CFG_KEY_ENCODER); if !force_reset && config.is_ok() { - config.unwrap() + (config.unwrap(), false) } else { let ctx = EncodeContext { name: String::from(""), @@ -157,10 +170,12 @@ impl HwEncoder { rc: DEFAULT_RC, }; let encoders = CodecInfo::score(Encoder::avaliable_encoders(ctx)); - let _ = set_config(key, &encoders) - .map_err(|e| log::error!("{:?}", e)) - .ok(); - encoders + if write { + let _ = set_config(CFG_KEY_ENCODER, &encoders) + .map_err(|e| log::error!("{:?}", e)) + .ok(); + } + (encoders, true) } } @@ -234,23 +249,24 @@ pub struct HwDecoders { } impl HwDecoder { - /// H264, H265 decoder info with the highest score. - fn best(force_reset: bool) -> CodecInfos { - let key = "bestHwDecoders"; - let config = get_config(key); + /// See HwEncoder::best + fn best(force_reset: bool, write: bool) -> (CodecInfos, bool) { + let config = get_config(CFG_KEY_DECODER); if !force_reset && config.is_ok() { - config.unwrap() + (config.unwrap(), false) } else { let decoders = CodecInfo::score(Decoder::avaliable_decoders()); - set_config(key, &decoders) - .map_err(|e| log::error!("{:?}", e)) - .ok(); - decoders + if write { + set_config(CFG_KEY_DECODER, &decoders) + .map_err(|e| log::error!("{:?}", e)) + .ok(); + } + (decoders, true) } } pub fn new_decoders() -> HwDecoders { - let best = HwDecoder::best(false); + let (best, _) = HwDecoder::best(false, true); let mut h264: Option = None; let mut h265: Option = None; let mut fail = false; @@ -268,14 +284,7 @@ impl HwDecoder { } } if fail { - HwDecoder::best(true); - } - - if h264.is_some() { - log::info!("h264 decoder:{:?}", h264.as_ref().unwrap().info); - } - if h265.is_some() { - log::info!("h265 decoder:{:?}", h265.as_ref().unwrap().info); + HwDecoder::best(true, true); } HwDecoders { h264, h265 } } @@ -336,7 +345,7 @@ impl HwDecoderImage<'_> { } fn get_config(k: &str) -> ResultType { - let v = LocalConfig::get_option(k); + let v = HwCodecConfig::get_option(k); match CodecInfos::deserialize(&v) { Ok(v) => Ok(v), Err(_) => Err(anyhow!("Failed to get config:{}", k)), @@ -346,9 +355,26 @@ fn get_config(k: &str) -> ResultType { fn set_config(k: &str, v: &CodecInfos) -> ResultType<()> { match v.serialize() { Ok(v) => { - LocalConfig::set_option(k.to_owned(), v); + HwCodecConfig::set_option(k.to_owned(), v); Ok(()) } Err(_) => Err(anyhow!("Failed to set config:{}", k)), } } + +pub fn check_config() -> Option> { + let (encoders, update_encoders) = HwEncoder::best(false, false); + let (decoders, update_decoders) = HwDecoder::best(false, false); + if update_encoders || update_decoders { + if let Ok(encoders) = encoders.serialize() { + if let Ok(decoders) = decoders.serialize() { + return Some(HashMap::from([ + (CFG_KEY_ENCODER.to_owned(), encoders), + (CFG_KEY_DECODER.to_owned(), decoders), + ])); + } + } + log::error!("Failed to serialize codec info"); + } + None +} diff --git a/src/ipc.rs b/src/ipc.rs index 2388a7d9c..be9f6d87f 100644 --- a/src/ipc.rs +++ b/src/ipc.rs @@ -1,6 +1,8 @@ use crate::rendezvous_mediator::RendezvousMediator; #[cfg(not(any(target_os = "android", target_os = "ios")))] pub use clipboard::ClipbaordFile; +#[cfg(feature = "hwcodec")] +use hbb_common::config::HwCodecConfig; use hbb_common::{ allow_err, bail, bytes, bytes_codec::BytesCodec, @@ -63,7 +65,7 @@ pub enum FS { WriteOffset { id: i32, file_num: i32, - offset_blk: u32 + offset_blk: u32, }, CheckDigest { id: i32, @@ -116,6 +118,8 @@ pub enum Data { #[cfg(not(any(target_os = "android", target_os = "ios")))] ClipbaordFile(ClipbaordFile), ClipboardFileEnabled(bool), + #[cfg(feature = "hwcodec")] + HwCodecConfig(Option>), } #[tokio::main(flavor = "current_thread")] @@ -325,6 +329,12 @@ async fn handle(data: Data, stream: &mut Connection) { .await ); } + #[cfg(feature = "hwcodec")] + Data::HwCodecConfig(Some(config)) => { + for (k, v) in config { + HwCodecConfig::set_option(k, v); + } + } _ => {} } } @@ -624,3 +634,20 @@ pub async fn set_socks(value: config::Socks5Server) -> ResultType<()> { .await?; Ok(()) } + +#[cfg(feature = "hwcodec")] +#[tokio::main] +pub async fn check_hwcodec_config() { + if let Some(config) = scrap::hwcodec::check_config() { + match connect(1000, "").await { + Ok(mut conn) => { + if conn.send(&Data::HwCodecConfig(Some(config))).await.is_err() { + log::error!("Failed to send hwcodec config by ipc"); + } + } + Err(err) => { + log::info!("Failed to connect ipc: {:?}", err); + } + } + } +} diff --git a/src/main.rs b/src/main.rs index d802b8ae9..16c365f5d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -70,6 +70,15 @@ fn main() { } if args.is_empty() { std::thread::spawn(move || start_server(false)); + #[cfg(feature = "hwcodec")] + if let Ok(exe) = std::env::current_exe() { + std::thread::spawn(move || { + std::process::Command::new(exe) + .arg("--check-hwcodec-config") + .status() + .ok() + }); + } } else { #[cfg(windows)] { @@ -104,6 +113,10 @@ fn main() { "".to_owned() )); return; + } else if args[0] == "--check-hwcodec-config" { + #[cfg(feature = "hwcodec")] + ipc::check_hwcodec_config(); + return; } } if args[0] == "--remove" {