Merge pull request #5282 from 21pages/record

enable keyframe interval when recording

flutter/lib/models/model.dart

@@ -1532,12 +1532,13 @@ class RecordingModel with ChangeNotifier {
         sessionId: sessionId, start: true, width: width, height: height);
   }
 
-  toggle() {
+  toggle() async {
     if (isIOS) return;
     final sessionId = parent.target?.sessionId;
     if (sessionId == null) return;
     _start = !_start;
     notifyListeners();
+    await bind.sessionRecordStatus(sessionId: sessionId, status: _start);
     if (_start) {
       bind.sessionRefresh(sessionId: sessionId);
     } else {

libs/hbb_common/protos/message.proto

@@ -664,6 +664,7 @@ message Misc {
     PluginFailure plugin_failure = 26;
     uint32 full_speed_fps = 27;
     uint32 auto_adjust_fps = 28;
+    bool client_record_status = 29;
   }
 }
 

libs/scrap/examples/benchmark.rs

@@ -114,9 +114,9 @@ fn test_vpx(
     let config = EncoderCfg::VPX(VpxEncoderConfig {
         width: width as _,
         height: height as _,
-        timebase: [1, 1000],
         quality,
         codec: codec_id,
+        keyframe_interval: None,
     });
     let mut encoder = VpxEncoder::new(config).unwrap();
     let mut vpxs = vec![];
@@ -161,6 +161,7 @@ fn test_av1(yuvs: &Vec<Vec<u8>>, width: usize, height: usize, quality: Q, yuv_co
         width: width as _,
         height: height as _,
         quality,
+        keyframe_interval: None,
     });
     let mut encoder = AomEncoder::new(config).unwrap();
     let start = Instant::now();

libs/scrap/examples/record-screen.rs

@@ -113,9 +113,9 @@ fn main() -> io::Result<()> {
     let mut vpx = vpx_encode::VpxEncoder::new(EncoderCfg::VPX(vpx_encode::VpxEncoderConfig {
         width,
         height,
-        timebase: [1, 1000],
         quality,
         codec: vpx_codec,
+        keyframe_interval: None,
     }))
     .unwrap();
 

libs/scrap/src/common/aom.rs

@@ -45,6 +45,7 @@ pub struct AomEncoderConfig {
     pub width: u32,
     pub height: u32,
     pub quality: Quality,
+    pub keyframe_interval: Option<usize>,
 }
 
 pub struct AomEncoder {
@@ -105,7 +106,12 @@ mod webrtc {
         c.g_timebase.num = 1;
         c.g_timebase.den = kRtpTicksPerSecond;
         c.g_input_bit_depth = kBitDepth;
-        c.kf_mode = aom_kf_mode::AOM_KF_DISABLED;
+        if let Some(keyframe_interval) = cfg.keyframe_interval {
+            c.kf_min_dist = 0;
+            c.kf_max_dist = keyframe_interval as _;
+        } else {
+            c.kf_mode = aom_kf_mode::AOM_KF_DISABLED;
+        }
         let (q_min, q_max, b) = AomEncoder::convert_quality(cfg.quality);
         if q_min > 0 && q_min < q_max && q_max < 64 {
             c.rc_min_quantizer = q_min;

libs/scrap/src/common/codec.rs

@@ -45,6 +45,7 @@ pub struct HwEncoderConfig {
     pub width: usize,
     pub height: usize,
     pub quality: Quality,
+    pub keyframe_interval: Option<usize>,
 }
 
 #[derive(Debug, Clone)]

libs/scrap/src/common/hwcodec.rs

@@ -52,6 +52,7 @@ impl EncoderApi for HwEncoder {
                 if base_bitrate <= 0 {
                     bitrate = base_bitrate;
                 }
+                let gop = config.keyframe_interval.unwrap_or(DEFAULT_GOP as _) as i32;
                 let ctx = EncodeContext {
                     name: config.name.clone(),
                     width: config.width as _,
@@ -60,7 +61,7 @@ impl EncoderApi for HwEncoder {
                     align: HW_STRIDE_ALIGN as _,
                     bitrate: bitrate as i32 * 1000,
                     timebase: DEFAULT_TIME_BASE,
-                    gop: DEFAULT_GOP,
+                    gop,
                     quality: DEFAULT_HW_QUALITY,
                     rc: DEFAULT_RC,
                     thread_count: codec_thread_num() as _, // ffmpeg's thread_count is used for cpu

libs/scrap/src/common/vpxcodec.rs

@@ -65,8 +65,8 @@ impl EncoderApi for VpxEncoder {
 
                 c.g_w = config.width;
                 c.g_h = config.height;
-                c.g_timebase.num = config.timebase[0];
+                c.g_timebase.num = 1;
-                c.g_timebase.den = config.timebase[1];
+                c.g_timebase.den = 1000; // Output timestamp precision
                 c.rc_undershoot_pct = 95;
                 // When the data buffer falls below this percentage of fullness, a dropped frame is indicated. Set the threshold to zero (0) to disable this feature.
                 // In dynamic scenes, low bitrate gets low fps while high bitrate gets high fps.
@@ -76,9 +76,13 @@ impl EncoderApi for VpxEncoder {
                 // https://developers.google.com/media/vp9/bitrate-modes/
                 // Constant Bitrate mode (CBR) is recommended for live streaming with VP9.
                 c.rc_end_usage = vpx_rc_mode::VPX_CBR;
-                // c.kf_min_dist = 0;
+                if let Some(keyframe_interval) = config.keyframe_interval {
-                // c.kf_max_dist = 999999;
+                    c.kf_min_dist = 0;
-                c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot
+                    c.kf_max_dist = keyframe_interval as _;
+                } else {
+                    c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot
+                }
+
                 let (q_min, q_max, b) = Self::convert_quality(config.quality);
                 if q_min > 0 && q_min < q_max && q_max < 64 {
                     c.rc_min_quantizer = q_min;
@@ -343,12 +347,12 @@ pub struct VpxEncoderConfig {
     pub width: c_uint,
     /// The height (in pixels).
     pub height: c_uint,
-    /// The timebase numerator and denominator (in seconds).
-    pub timebase: [c_int; 2],
     /// The image quality
     pub quality: Quality,
     /// The codec
     pub codec: VpxVideoCodecId,
+    /// keyframe interval
+    pub keyframe_interval: Option<usize>,
 }
 
 #[derive(Clone, Copy, Debug)]

src/flutter_ffi.rs

@@ -174,6 +174,12 @@ pub fn session_record_screen(session_id: SessionID, start: bool, width: usize, h
     }
 }
 
+pub fn session_record_status(session_id: SessionID, status: bool) {
+    if let Some(session) = SESSIONS.read().unwrap().get(&session_id) {
+        session.record_status(status);
+    }
+}
+
 pub fn session_reconnect(session_id: SessionID, force_relay: bool) {
     if let Some(session) = SESSIONS.read().unwrap().get(&session_id) {
         session.reconnect(force_relay);

src/server/connection.rs

@@ -1907,6 +1907,10 @@ impl Connection {
                         .lock()
                         .unwrap()
                         .user_auto_adjust_fps(self.inner.id(), fps),
+                    Some(misc::Union::ClientRecordStatus(status)) => video_service::VIDEO_QOS
+                        .lock()
+                        .unwrap()
+                        .user_record(self.inner.id(), status),
                     _ => {}
                 },
                 Some(message::Union::AudioFrame(frame)) => {

src/server/video_qos.rs

@@ -36,6 +36,7 @@ struct UserData {
     quality: Option<(i64, Quality)>, // (time, quality)
     delay: Option<Delay>,
     response_delayed: bool,
+    record: bool,
 }
 
 pub struct VideoQoS {
@@ -114,6 +115,10 @@ impl VideoQoS {
         self.quality
     }
 
+    pub fn record(&self) -> bool {
+        self.users.iter().any(|u| u.1.record)
+    }
+
     pub fn abr_enabled() -> bool {
         "N" != Config::get_option("enable-abr")
     }
@@ -388,6 +393,12 @@ impl VideoQoS {
         }
     }
 
+    pub fn user_record(&mut self, id: i32, v: bool) {
+        if let Some(user) = self.users.get_mut(&id) {
+            user.record = v;
+        }
+    }
+
     pub fn on_connection_close(&mut self, id: i32) {
         self.users.remove(&id);
         self.refresh(None);

src/server/video_service.rs

@@ -36,7 +36,7 @@ use hbb_common::{
 use scrap::Capturer;
 use scrap::{
     aom::AomEncoderConfig,
-    codec::{Encoder, EncoderCfg, HwEncoderConfig},
+    codec::{Encoder, EncoderCfg, HwEncoderConfig, Quality},
     record::{Recorder, RecorderContext},
     vpxcodec::{VpxEncoderConfig, VpxVideoCodecId},
     CodecName, Display, TraitCapturer,
@@ -518,37 +518,13 @@ fn run(sp: GenericService) -> ResultType<()> {
     let mut spf;
     let mut quality = video_qos.quality();
     let abr = VideoQoS::abr_enabled();
-    drop(video_qos);
     log::info!("init quality={:?}, abr enabled:{}", quality, abr);
-
+    let codec_name = Encoder::negotiated_codec();
-    let encoder_cfg = match Encoder::negotiated_codec() {
+    let recorder = get_recorder(c.width, c.height, &codec_name);
-        scrap::CodecName::H264(name) | scrap::CodecName::H265(name) => {
+    let last_recording =
-            EncoderCfg::HW(HwEncoderConfig {
+        (recorder.lock().unwrap().is_some() || video_qos.record()) && codec_name != CodecName::AV1;
-                name,
+    drop(video_qos);
-                width: c.width,
+    let encoder_cfg = get_encoder_config(&c, quality, last_recording);
-                height: c.height,
-                quality,
-            })
-        }
-        name @ (scrap::CodecName::VP8 | scrap::CodecName::VP9) => {
-            EncoderCfg::VPX(VpxEncoderConfig {
-                width: c.width as _,
-                height: c.height as _,
-                timebase: [1, 1000], // Output timestamp precision
-                quality,
-                codec: if name == scrap::CodecName::VP8 {
-                    VpxVideoCodecId::VP8
-                } else {
-                    VpxVideoCodecId::VP9
-                },
-            })
-        }
-        scrap::CodecName::AV1 => EncoderCfg::AOM(AomEncoderConfig {
-            width: c.width as _,
-            height: c.height as _,
-            quality,
-        }),
-    };
 
     let mut encoder;
     match Encoder::new(encoder_cfg) {
@@ -597,8 +573,6 @@ fn run(sp: GenericService) -> ResultType<()> {
     let mut try_gdi = 1;
     #[cfg(windows)]
     log::info!("gdi: {}", c.is_gdi());
-    let codec_name = Encoder::negotiated_codec();
-    let recorder = get_recorder(c.width, c.height, &codec_name);
     #[cfg(windows)]
     start_uac_elevation_check();
 
@@ -617,6 +591,11 @@ fn run(sp: GenericService) -> ResultType<()> {
             allow_err!(encoder.set_quality(quality));
             video_qos.store_bitrate(encoder.bitrate());
         }
+        let recording = (recorder.lock().unwrap().is_some() || video_qos.record())
+            && codec_name != CodecName::AV1;
+        if recording != last_recording {
+            bail!("SWITCH");
+        }
         drop(video_qos);
 
         if *SWITCH.lock().unwrap() {
@@ -789,6 +768,41 @@ fn run(sp: GenericService) -> ResultType<()> {
     Ok(())
 }
 
+fn get_encoder_config(c: &CapturerInfo, quality: Quality, recording: bool) -> EncoderCfg {
+    // https://www.wowza.com/community/t/the-correct-keyframe-interval-in-obs-studio/95162
+    let keyframe_interval = if recording { Some(240) } else { None };
+    match Encoder::negotiated_codec() {
+        scrap::CodecName::H264(name) | scrap::CodecName::H265(name) => {
+            EncoderCfg::HW(HwEncoderConfig {
+                name,
+                width: c.width,
+                height: c.height,
+                quality,
+                keyframe_interval,
+            })
+        }
+        name @ (scrap::CodecName::VP8 | scrap::CodecName::VP9) => {
+            EncoderCfg::VPX(VpxEncoderConfig {
+                width: c.width as _,
+                height: c.height as _,
+                quality,
+                codec: if name == scrap::CodecName::VP8 {
+                    VpxVideoCodecId::VP8
+                } else {
+                    VpxVideoCodecId::VP9
+                },
+                keyframe_interval,
+            })
+        }
+        scrap::CodecName::AV1 => EncoderCfg::AOM(AomEncoderConfig {
+            width: c.width as _,
+            height: c.height as _,
+            quality,
+            keyframe_interval,
+        }),
+    }
+}
+
 fn get_recorder(
     width: usize,
     height: usize,

src/ui/header.tis

@@ -297,6 +297,7 @@ class Header: Reactor.Component {
     event click $(span#recording) (_, me) {
         recording = !recording;
         header.update();
+        handler.record_status(recording);
         if (recording)
             handler.refresh_video();
         else

src/ui/remote.rs

@@ -451,6 +451,7 @@ impl sciter::EventHandler for SciterSession {
         fn save_custom_image_quality(i32);
         fn refresh_video();
         fn record_screen(bool, i32, i32);
+        fn record_status(bool);
         fn get_toggle_option(String);
         fn is_privacy_mode_supported();
         fn toggle_option(String);

src/ui_session_interface.rs

@@ -240,6 +240,14 @@ impl<T: InvokeUiSession> Session<T> {
         self.send(Data::RecordScreen(start, w, h, self.id.clone()));
     }
 
+    pub fn record_status(&self, status: bool) {
+        let mut misc = Misc::new();
+        misc.set_client_record_status(status);
+        let mut msg = Message::new();
+        msg.set_misc(misc);
+        self.send(Data::Message(msg));
+    }
+
     pub fn save_custom_image_quality(&mut self, custom_image_quality: i32) {
         let msg = self
             .lc