diff --git a/Cargo.lock b/Cargo.lock index aa18fb9ce..b48763554 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3037,8 +3037,8 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hwcodec" -version = "0.4.9" -source = "git+https://github.com/21pages/hwcodec#7a52282267cb6aadbcd74c132bd4ecd43ab6f505" +version = "0.4.10" +source = "git+https://github.com/21pages/hwcodec#9cb895fdaea198dd72bd75980109dbd05a059a60" dependencies = [ "bindgen 0.59.2", "cc", diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt index 52a5ff75e..ab51b20d1 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt @@ -7,6 +7,8 @@ package com.carriez.flutter_hbb * Inspired by [droidVNC-NG] https://github.com/bk138/droidVNC-NG */ +import ffi.FFI + import android.content.ComponentName import android.content.Context import android.content.Intent @@ -15,10 +17,20 @@ import android.os.Build import android.os.IBinder import android.util.Log import android.view.WindowManager +import android.media.MediaCodecInfo +import android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface +import android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar +import android.media.MediaCodecList +import android.media.MediaFormat +import android.util.DisplayMetrics +import androidx.annotation.RequiresApi +import org.json.JSONArray +import org.json.JSONObject import com.hjq.permissions.XXPermissions import io.flutter.embedding.android.FlutterActivity import io.flutter.embedding.engine.FlutterEngine import io.flutter.plugin.common.MethodChannel +import kotlin.concurrent.thread class MainActivity : FlutterActivity() { @@ -42,6 +54,7 @@ class MainActivity : FlutterActivity() { channelTag ) initFlutterChannel(flutterMethodChannel!!) + thread { setCodecInfo() } } override fun onResume() { @@ -223,4 +236,80 @@ class MainActivity : FlutterActivity() { } } } + + private fun setCodecInfo() { + val codecList = MediaCodecList(MediaCodecList.REGULAR_CODECS) + val codecs = codecList.codecInfos + val codecArray = JSONArray() + + val windowManager = getSystemService(Context.WINDOW_SERVICE) as WindowManager + var w = 0 + var h = 0 + @Suppress("DEPRECATION") + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + val m = windowManager.maximumWindowMetrics + w = m.bounds.width() + h = m.bounds.height() + } else { + val dm = DisplayMetrics() + windowManager.defaultDisplay.getRealMetrics(dm) + w = dm.widthPixels + h = dm.heightPixels + } + codecs.forEach { codec -> + val codecObject = JSONObject() + codecObject.put("name", codec.name) + codecObject.put("is_encoder", codec.isEncoder) + var hw: Boolean? = null; + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + hw = codec.isHardwareAccelerated + } else { + if (listOf("OMX.google.", "OMX.SEC.", "c2.android").any { codec.name.startsWith(it) }) { + hw = false + } + } + codecObject.put("hw", hw) + var mime_type = "" + codec.supportedTypes.forEach { type -> + if (listOf("video/avc", "video/hevc", "video/x-vnd.on2.vp8", "video/x-vnd.on2.vp9", "video/av01").contains(type)) { + mime_type = type; + } + } + if (mime_type.isNotEmpty()) { + codecObject.put("mime_type", mime_type) + val caps = codec.getCapabilitiesForType(mime_type) + var usable = true; + if (codec.isEncoder) { + if (!caps.videoCapabilities.isSizeSupported(w,h) || !caps.videoCapabilities.isSizeSupported(h,w)) { + usable = false + } + } + codecObject.put("min_width", caps.videoCapabilities.supportedWidths.lower) + codecObject.put("max_width", caps.videoCapabilities.supportedWidths.upper) + codecObject.put("min_height", caps.videoCapabilities.supportedHeights.lower) + codecObject.put("max_height", caps.videoCapabilities.supportedHeights.upper) + val surface = caps.colorFormats.contains(COLOR_FormatSurface); + codecObject.put("surface", surface) + val nv12 = caps.colorFormats.contains(COLOR_FormatYUV420SemiPlanar) + codecObject.put("nv12", nv12) + if (!(nv12 || surface)) { + usable = false + } + codecObject.put("min_bitrate", caps.videoCapabilities.bitrateRange.lower / 1000) + codecObject.put("max_bitrate", caps.videoCapabilities.bitrateRange.upper / 1000) + if (!codec.isEncoder) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + codecObject.put("low_latency", caps.isFeatureSupported(MediaCodecInfo.CodecCapabilities.FEATURE_LowLatency)) + } + } + if (usable) { + codecArray.put(codecObject) + } + } + } + val result = JSONObject() + result.put("version", Build.VERSION.SDK_INT) + result.put("codecs", codecArray) + FFI.setCodecInfo(result.toString()) + } } diff --git a/flutter/android/app/src/main/kotlin/ffi.kt b/flutter/android/app/src/main/kotlin/ffi.kt index ba29021b3..aa26b36a6 100644 --- a/flutter/android/app/src/main/kotlin/ffi.kt +++ b/flutter/android/app/src/main/kotlin/ffi.kt @@ -18,4 +18,5 @@ object FFI { external fun translateLocale(localeName: String, input: String): String external fun refreshScreen() external fun setFrameRawEnable(name: String, value: Boolean) + external fun setCodecInfo(info: String) } \ No newline at end of file diff --git a/flutter/lib/mobile/pages/settings_page.dart b/flutter/lib/mobile/pages/settings_page.dart index fff7e3516..48c150a40 100644 --- a/flutter/lib/mobile/pages/settings_page.dart +++ b/flutter/lib/mobile/pages/settings_page.dart @@ -44,6 +44,7 @@ class _SettingsState extends State with WidgetsBindingObserver { var _onlyWhiteList = false; var _enableDirectIPAccess = false; var _enableRecordSession = false; + var _enableHardwareCodec = false; var _autoRecordIncomingSession = false; var _allowAutoDisconnect = false; var _localIP = ""; @@ -120,6 +121,13 @@ class _SettingsState extends State with WidgetsBindingObserver { _enableRecordSession = enableRecordSession; } + final enableHardwareCodec = option2bool( + 'enable-hwcodec', await bind.mainGetOption(key: 'enable-hwcodec')); + if (_enableHardwareCodec != enableHardwareCodec) { + update = true; + _enableHardwareCodec = enableHardwareCodec; + } + final autoRecordIncomingSession = option2bool( 'allow-auto-record-incoming', await bind.mainGetOption(key: 'allow-auto-record-incoming')); @@ -513,6 +521,22 @@ class _SettingsState extends State with WidgetsBindingObserver { }, ) ]), + if (isAndroid) + SettingsSection(title: Text(translate('Hardware Codec')), tiles: [ + SettingsTile.switchTile( + title: Text(translate('Enable hardware codec')), + initialValue: _enableHardwareCodec, + onToggle: (v) async { + await bind.mainSetOption( + key: "enable-hwcodec", value: v ? "" : "N"); + final newValue = + await bind.mainGetOption(key: "enable-hwcodec") != "N"; + setState(() { + _enableHardwareCodec = newValue; + }); + }, + ), + ]), if (isAndroid && !outgoingOnly) SettingsSection( title: Text(translate("Recording")), diff --git a/libs/scrap/examples/benchmark.rs b/libs/scrap/examples/benchmark.rs index eb74bfad4..3ef7916a9 100644 --- a/libs/scrap/examples/benchmark.rs +++ b/libs/scrap/examples/benchmark.rs @@ -272,6 +272,7 @@ mod hw { let mut encoder = HwRamEncoder::new( EncoderCfg::HWRAM(HwRamEncoderConfig { name: info.name.clone(), + mc_name: None, width, height, quality, diff --git a/libs/scrap/src/android/ffi.rs b/libs/scrap/src/android/ffi.rs index 7079cb3b9..a604166a7 100644 --- a/libs/scrap/src/android/ffi.rs +++ b/libs/scrap/src/android/ffi.rs @@ -10,6 +10,7 @@ use jni::{ use jni::errors::{Error as JniError, Result as JniResult}; use lazy_static::lazy_static; +use serde::Deserialize; use std::ops::Not; use std::sync::atomic::{AtomicPtr, Ordering::SeqCst}; use std::sync::{Mutex, RwLock}; @@ -20,6 +21,7 @@ lazy_static! { static ref VIDEO_RAW: Mutex = Mutex::new(FrameRaw::new("video", MAX_VIDEO_FRAME_TIMEOUT)); static ref AUDIO_RAW: Mutex = Mutex::new(FrameRaw::new("audio", MAX_AUDIO_FRAME_TIMEOUT)); static ref NDK_CONTEXT_INITED: Mutex = Default::default(); + static ref MEDIA_CODEC_INFOS: RwLock> = RwLock::new(None); } const MAX_VIDEO_FRAME_TIMEOUT: Duration = Duration::from_millis(100); @@ -154,6 +156,50 @@ pub extern "system" fn Java_ffi_FFI_init(env: JNIEnv, _class: JClass, ctx: JObje } } +#[derive(Debug, Deserialize, Clone)] +pub struct MediaCodecInfo { + pub name: String, + pub is_encoder: bool, + #[serde(default)] + pub hw: Option, // api 29+ + pub mime_type: String, + pub surface: bool, + pub nv12: bool, + #[serde(default)] + pub low_latency: Option, // api 30+, decoder + pub min_bitrate: u32, + pub max_bitrate: u32, + pub min_width: usize, + pub max_width: usize, + pub min_height: usize, + pub max_height: usize, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct MediaCodecInfos { + pub version: usize, + pub codecs: Vec, +} + +#[no_mangle] +pub extern "system" fn Java_ffi_FFI_setCodecInfo(env: JNIEnv, _class: JClass, info: JString) { + let mut env = env; + if let Ok(info) = env.get_string(&info) { + let info: String = info.into(); + if let Ok(infos) = serde_json::from_str::(&info) { + *MEDIA_CODEC_INFOS.write().unwrap() = Some(infos); + } + } +} + +pub fn get_codec_info() -> Option { + MEDIA_CODEC_INFOS.read().unwrap().as_ref().cloned() +} + +pub fn clear_codec_info() { + *MEDIA_CODEC_INFOS.write().unwrap() = None; +} + pub fn call_main_service_pointer_input(kind: &str, mask: i32, x: i32, y: i32) -> JniResult<()> { if let (Some(jvm), Some(ctx)) = ( JVM.read().unwrap().as_ref(), diff --git a/libs/scrap/src/common/aom.rs b/libs/scrap/src/common/aom.rs index 759bf3fbe..7d16eeafd 100644 --- a/libs/scrap/src/common/aom.rs +++ b/libs/scrap/src/common/aom.rs @@ -296,6 +296,14 @@ impl EncoderApi for AomEncoder { fn support_abr(&self) -> bool { true } + + fn support_changing_quality(&self) -> bool { + true + } + + fn latency_free(&self) -> bool { + true + } } impl AomEncoder { diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index b189dadac..5a04cfacc 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -70,6 +70,10 @@ pub trait EncoderApi { fn bitrate(&self) -> u32; fn support_abr(&self) -> bool; + + fn support_changing_quality(&self) -> bool; + + fn latency_free(&self) -> bool; } pub struct Encoder { @@ -138,6 +142,9 @@ impl Encoder { }), Err(e) => { log::error!("new hw encoder failed: {e:?}, clear config"); + #[cfg(target_os = "android")] + crate::android::ffi::clear_codec_info(); + #[cfg(not(target_os = "android"))] hbb_common::config::HwCodecConfig::clear_ram(); Self::update(EncodingUpdate::Check); *ENCODE_CODEC_FORMAT.lock().unwrap() = CodecFormat::VP9; @@ -346,7 +353,14 @@ impl Encoder { EncoderCfg::AOM(_) => CodecFormat::AV1, #[cfg(feature = "hwcodec")] EncoderCfg::HWRAM(hw) => { - if hw.name.to_lowercase().contains("h264") { + let name = hw.name.to_lowercase(); + if name.contains("vp8") { + CodecFormat::VP8 + } else if name.contains("vp9") { + CodecFormat::VP9 + } else if name.contains("av1") { + CodecFormat::AV1 + } else if name.contains("h264") { CodecFormat::H264 } else { CodecFormat::H265 @@ -817,7 +831,7 @@ impl Decoder { #[cfg(any(feature = "hwcodec", feature = "mediacodec"))] pub fn enable_hwcodec_option() -> bool { - if cfg!(windows) || cfg!(target_os = "linux") || cfg!(feature = "mediacodec") { + if cfg!(windows) || cfg!(target_os = "linux") || cfg!(target_os = "android") { if let Some(v) = Config2::get().options.get("enable-hwcodec") { return v != "N"; } @@ -847,6 +861,15 @@ impl Default for Quality { } } +impl Quality { + pub fn is_custom(&self) -> bool { + match self { + Quality::Custom(_) => true, + _ => false, + } + } +} + pub fn base_bitrate(width: u32, height: u32) -> u32 { #[allow(unused_mut)] let mut base_bitrate = ((width * height) / 1000) as u32; // same as 1.1.9 diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index 9c03937b5..a1dbeae18 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -29,11 +29,15 @@ const DEFAULT_PIXFMT: AVPixelFormat = AVPixelFormat::AV_PIX_FMT_NV12; pub const DEFAULT_TIME_BASE: [i32; 2] = [1, 30]; const DEFAULT_GOP: i32 = i32::MAX; const DEFAULT_HW_QUALITY: Quality = Quality_Default; -const DEFAULT_RC: RateControl = RC_DEFAULT; +#[cfg(target_os = "android")] +const DEFAULT_RC: RateControl = RC_VBR; // android cbr poor quality +#[cfg(not(target_os = "android"))] +const DEFAULT_RC: RateControl = RC_CBR; #[derive(Debug, Clone)] pub struct HwRamEncoderConfig { pub name: String, + pub mc_name: Option, pub width: usize, pub height: usize, pub quality: Q, @@ -57,20 +61,22 @@ impl EncoderApi for HwRamEncoder { { match cfg { EncoderCfg::HWRAM(config) => { - let b = Self::convert_quality(config.quality); + let b = Self::convert_quality(&config.name, config.quality); let base_bitrate = base_bitrate(config.width as _, config.height as _); let mut bitrate = base_bitrate * b / 100; if base_bitrate <= 0 { bitrate = base_bitrate; } + bitrate = Self::check_bitrate_range(&config.name, bitrate); let gop = config.keyframe_interval.unwrap_or(DEFAULT_GOP as _) as i32; let ctx = EncodeContext { name: config.name.clone(), + mc_name: config.mc_name.clone(), width: config.width as _, height: config.height as _, pixfmt: DEFAULT_PIXFMT, align: HW_STRIDE_ALIGN as _, - bitrate: bitrate as i32 * 1000, + kbs: bitrate as i32, timebase: DEFAULT_TIME_BASE, gop, quality: DEFAULT_HW_QUALITY, @@ -166,10 +172,11 @@ impl EncoderApi for HwRamEncoder { } fn set_quality(&mut self, quality: crate::codec::Quality) -> ResultType<()> { - let b = Self::convert_quality(quality); - let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100; + let b = Self::convert_quality(&self.name, quality); + let mut bitrate = base_bitrate(self.width as _, self.height as _) * b / 100; if bitrate > 0 { - self.encoder.set_bitrate((bitrate * 1000) as _).ok(); + bitrate = Self::check_bitrate_range(&self.name, bitrate); + self.encoder.set_bitrate(bitrate as _).ok(); self.bitrate = bitrate; } Ok(()) @@ -180,7 +187,19 @@ impl EncoderApi for HwRamEncoder { } fn support_abr(&self) -> bool { - ["qsv", "vaapi"].iter().all(|&x| !self.name.contains(x)) + ["qsv", "vaapi", "mediacodec"] + .iter() + .all(|&x| !self.name.contains(x)) + } + + fn support_changing_quality(&self) -> bool { + ["vaapi", "mediacodec"] + .iter() + .all(|&x| !self.name.contains(x)) + } + + fn latency_free(&self) -> bool { + !self.name.contains("mediacodec") } } @@ -217,14 +236,42 @@ impl HwRamEncoder { } } - pub fn convert_quality(quality: crate::codec::Quality) -> u32 { + pub fn convert_quality(name: &str, quality: crate::codec::Quality) -> u32 { use crate::codec::Quality; - match quality { + let quality = match quality { Quality::Best => 150, Quality::Balanced => 100, Quality::Low => 50, Quality::Custom(b) => b, + }; + let factor = if name.contains("mediacodec") { + if name.contains("h264") { + 6 + } else { + 3 + } + } else { + 1 + }; + quality * factor + } + + pub fn check_bitrate_range(name: &str, bitrate: u32) -> u32 { + #[cfg(target_os = "android")] + if name.contains("mediacodec") { + let info = crate::android::ffi::get_codec_info(); + if let Some(info) = info { + if let Some(codec) = info.codecs.iter().find(|c| c.name == name && c.is_encoder) { + if bitrate > codec.max_bitrate { + return codec.max_bitrate; + } + if bitrate < codec.min_bitrate { + return codec.min_bitrate; + } + } + } } + bitrate } } @@ -285,7 +332,10 @@ impl HwRamDecoder { match Decoder::new(ctx) { Ok(decoder) => Ok(HwRamDecoder { decoder, info }), Err(_) => { - HwCodecConfig::clear_ram(); + #[cfg(target_os = "android")] + crate::android::ffi::clear_codec_info(); + #[cfg(not(target_os = "android"))] + hbb_common::config::HwCodecConfig::clear_ram(); Err(anyhow!(format!("Failed to create decoder"))) } } @@ -363,20 +413,87 @@ struct Available { } fn get_config() -> ResultType { - match serde_json::from_str(&HwCodecConfig::load().ram) { - Ok(v) => Ok(v), - Err(e) => Err(anyhow!("Failed to get config:{e:?}")), + #[cfg(target_os = "android")] + { + let info = crate::android::ffi::get_codec_info(); + struct T { + name_prefix: &'static str, + data_format: DataFormat, + } + let ts = vec![ + T { + name_prefix: "h264", + data_format: DataFormat::H264, + }, + T { + name_prefix: "hevc", + data_format: DataFormat::H265, + }, + ]; + let mut e = vec![]; + if let Some(info) = info { + ts.iter().for_each(|t| { + let codecs: Vec<_> = info + .codecs + .iter() + .filter(|c| { + c.is_encoder + && c.mime_type.as_str() == get_mime_type(t.data_format) + && c.nv12 + }) + .collect(); + log::debug!("available {:?} encoders: {codecs:?}", t.data_format); + let mut best = None; + if let Some(c) = codecs.iter().find(|c| c.hw == Some(true)) { + best = Some(c.name.clone()); + } else if let Some(c) = codecs.iter().find(|c| c.hw == None) { + best = Some(c.name.clone()); + } else if let Some(c) = codecs.first() { + best = Some(c.name.clone()); + } + if let Some(best) = best { + e.push(CodecInfo { + name: format!("{}_mediacodec", t.name_prefix), + mc_name: Some(best), + format: t.data_format, + hwdevice: hwcodec::ffmpeg::AVHWDeviceType::AV_HWDEVICE_TYPE_NONE, + priority: 0, + }); + } + }); + } + log::debug!("e: {e:?}"); + Ok(Available { e, d: vec![] }) + } + #[cfg(not(target_os = "android"))] + { + match serde_json::from_str(&HwCodecConfig::load().ram) { + Ok(v) => Ok(v), + Err(e) => Err(anyhow!("Failed to get config:{e:?}")), + } + } +} + +#[cfg(target_os = "android")] +fn get_mime_type(codec: DataFormat) -> &'static str { + match codec { + DataFormat::VP8 => "video/x-vnd.on2.vp8", + DataFormat::VP9 => "video/x-vnd.on2.vp9", + DataFormat::AV1 => "video/av01", + DataFormat::H264 => "video/avc", + DataFormat::H265 => "video/hevc", } } pub fn check_available_hwcodec() { let ctx = EncodeContext { name: String::from(""), + mc_name: None, width: 1280, height: 720, pixfmt: DEFAULT_PIXFMT, align: HW_STRIDE_ALIGN as _, - bitrate: 0, + kbs: 0, timebase: DEFAULT_TIME_BASE, gop: DEFAULT_GOP, quality: DEFAULT_HW_QUALITY, diff --git a/libs/scrap/src/common/vpxcodec.rs b/libs/scrap/src/common/vpxcodec.rs index a3e7e99e6..91507f053 100644 --- a/libs/scrap/src/common/vpxcodec.rs +++ b/libs/scrap/src/common/vpxcodec.rs @@ -235,6 +235,13 @@ impl EncoderApi for VpxEncoder { fn support_abr(&self) -> bool { true } + fn support_changing_quality(&self) -> bool { + true + } + + fn latency_free(&self) -> bool { + true + } } impl VpxEncoder { diff --git a/libs/scrap/src/common/vram.rs b/libs/scrap/src/common/vram.rs index 3b4113fd1..f85f15ed8 100644 --- a/libs/scrap/src/common/vram.rs +++ b/libs/scrap/src/common/vram.rs @@ -180,6 +180,14 @@ impl EncoderApi for VRamEncoder { fn support_abr(&self) -> bool { self.config.device.vendor_id != ADAPTER_VENDOR_INTEL as u32 } + + fn support_changing_quality(&self) -> bool { + true + } + + fn latency_free(&self) -> bool { + true + } } impl VRamEncoder { diff --git a/src/server/video_qos.rs b/src/server/video_qos.rs index 3e7ca8fc6..5d3aeca85 100644 --- a/src/server/video_qos.rs +++ b/src/server/video_qos.rs @@ -124,6 +124,10 @@ impl VideoQoS { self.support_abr.insert(display_idx, support); } + pub fn in_vbr_state(&self) -> bool { + Config::get_option("enable-abr") != "N" && self.support_abr.iter().all(|e| *e.1) + } + pub fn refresh(&mut self, typ: Option) { // fps let user_fps = |u: &UserData| { @@ -178,8 +182,7 @@ impl VideoQoS { let mut quality = latest_quality; // network delay - let abr_enabled = - Config::get_option("enable-abr") != "N" && self.support_abr.iter().all(|e| *e.1); + let abr_enabled = self.in_vbr_state(); if abr_enabled && typ != Some(RefreshType::SetImageQuality) { // max delay let delay = self diff --git a/src/server/video_service.rs b/src/server/video_service.rs index 53f9b1b21..902b5d147 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -53,7 +53,7 @@ use scrap::{ codec::{Encoder, EncoderCfg, Quality}, record::{Recorder, RecorderContext}, vpxcodec::{VpxEncoderConfig, VpxVideoCodecId}, - CodecFormat, Display, Frame, TraitCapturer, + CodecFormat, Display, EncodeInput, Frame, TraitCapturer, }; #[cfg(windows)] use std::sync::Once; @@ -470,6 +470,8 @@ fn run(vs: VideoService) -> ResultType<()> { let mut would_block_count = 0u32; let mut yuv = Vec::new(); let mut mid_data = Vec::new(); + let mut repeat_encode_counter = 0; + let repeat_encode_max = 10; while sp.ok() { #[cfg(windows)] @@ -480,8 +482,15 @@ fn run(vs: VideoService) -> ResultType<()> { if quality != video_qos.quality() { log::debug!("quality: {:?} -> {:?}", quality, video_qos.quality()); quality = video_qos.quality(); - allow_err!(encoder.set_quality(quality)); - video_qos.store_bitrate(encoder.bitrate()); + if encoder.support_changing_quality() { + allow_err!(encoder.set_quality(quality)); + video_qos.store_bitrate(encoder.bitrate()); + } else { + if !video_qos.in_vbr_state() && !quality.is_custom() { + log::info!("switch to change quality"); + bail!("SWITCH"); + } + } } if client_record != video_qos.record() { bail!("SWITCH"); @@ -526,17 +535,17 @@ fn run(vs: VideoService) -> ResultType<()> { frame_controller.reset(); + let time = now - start; + let ms = (time.as_secs() * 1000 + time.subsec_millis() as u64) as i64; let res = match c.frame(spf) { Ok(frame) => { - let time = now - start; - let ms = (time.as_secs() * 1000 + time.subsec_millis() as u64) as i64; + repeat_encode_counter = 0; if frame.valid() { + let frame = frame.to(encoder.yuvfmt(), &mut yuv, &mut mid_data)?; let send_conn_ids = handle_one_frame( display_idx, &sp, frame, - &mut yuv, - &mut mid_data, ms, &mut encoder, recorder.clone(), @@ -579,6 +588,21 @@ fn run(vs: VideoService) -> ResultType<()> { } } } + if !encoder.latency_free() && yuv.len() > 0 { + // yun.len() > 0 means the frame is not texture. + if repeat_encode_counter < repeat_encode_max { + repeat_encode_counter += 1; + let send_conn_ids = handle_one_frame( + display_idx, + &sp, + EncodeInput::YUV(&yuv), + ms, + &mut encoder, + recorder.clone(), + )?; + frame_controller.set_send(now, send_conn_ids); + } + } } Err(err) => { // Get display information again immediately after error. @@ -707,6 +731,7 @@ fn get_encoder_config( if let Some(hw) = HwRamEncoder::try_get(negotiated_codec) { return EncoderCfg::HWRAM(HwRamEncoderConfig { name: hw.name, + mc_name: hw.mc_name, width: c.width, height: c.height, quality, @@ -805,9 +830,7 @@ fn check_privacy_mode_changed(sp: &GenericService, privacy_mode_id: i32) -> Resu fn handle_one_frame( display: usize, sp: &GenericService, - frame: Frame, - yuv: &mut Vec, - mid_data: &mut Vec, + frame: EncodeInput, ms: i64, encoder: &mut Encoder, recorder: Arc>>, @@ -820,7 +843,6 @@ fn handle_one_frame( Ok(()) })?; - let frame = frame.to(encoder.yuvfmt(), yuv, mid_data)?; let mut send_conn_ids: HashSet = Default::default(); match encoder.encode_to_message(frame, ms) { Ok(mut vf) => {