abr

libs/hbb_common/protos/message.proto

@@ -425,9 +425,9 @@ message PermissionInfo {
 
 enum ImageQuality {
   NotSet = 0;
-  Low = 2;
+  Low = 50;
-  Balanced = 3;
+  Balanced = 66;
-  Best = 4;
+  Best = 100;
 }
 
 message OptionMessage {
@@ -441,15 +441,18 @@ message OptionMessage {
   BoolOption show_remote_cursor = 3;
   BoolOption privacy_mode = 4;
   BoolOption block_input = 5;
-  int32 custom_image_quality = 6;
+  uint32 custom_image_quality = 6;
   BoolOption disable_audio = 7;
   BoolOption disable_clipboard = 8;
   BoolOption enable_file_transfer = 9;
+  BoolOption enable_abr = 10;
 }
 
 message TestDelay {
   int64 time = 1;
   bool from_client = 2;
+  uint32 last_delay = 3;
+  uint32 target_bitrate = 4;
 }
 
 message PublicKey {

libs/hbb_common/src/config.rs

@@ -139,6 +139,8 @@ pub struct PeerConfig {
     pub disable_clipboard: bool,
     #[serde(default)]
     pub enable_file_transfer: bool,
+    #[serde(default)]
+    pub show_quality_monitor: bool,
 
     // the other scalar value must before this
     #[serde(default)]

libs/scrap/src/common/android.rs

@@ -3,8 +3,8 @@ use crate::rgba_to_i420;
 use lazy_static::lazy_static;
 use serde_json::Value;
 use std::collections::HashMap;
-use std::io;
 use std::sync::Mutex;
+use std::{io, time::Duration};
 
 lazy_static! {
     static ref SCREEN_SIZE: Mutex<(u16, u16, u16)> = Mutex::new((0, 0, 0)); // (width, height, scale)
@@ -33,7 +33,7 @@ impl Capturer {
         self.display.height() as usize
     }
 
-    pub fn frame<'a>(&'a mut self, _timeout_ms: u32) -> io::Result<Frame<'a>> {
+    pub fn frame<'a>(&'a mut self, _timeout: Duration) -> io::Result<Frame<'a>> {
         if let Some(buf) = get_video_raw() {
             crate::would_block_if_equal(&mut self.saved_raw_data, buf)?;
             rgba_to_i420(self.width(), self.height(), buf, &mut self.bgra);

libs/scrap/src/common/codec.rs

@@ -107,17 +107,6 @@ impl Encoder {
         c.rc_target_bitrate = config.bitrate;
         c.rc_undershoot_pct = 95;
         c.rc_dropframe_thresh = 25;
-        if config.rc_min_quantizer > 0 {
-            c.rc_min_quantizer = config.rc_min_quantizer;
-        }
-        if config.rc_max_quantizer > 0 {
-            c.rc_max_quantizer = config.rc_max_quantizer;
-        }
-        let mut speed = config.speed;
-        if speed <= 0 {
-            speed = 6;
-        }
-
         c.g_threads = if num_threads == 0 {
             num_cpus::get() as _
         } else {
@@ -162,7 +151,7 @@ impl Encoder {
             Higher numbers (7 or 8) will be lower quality but more manageable for lower latency
             use cases and also for lower CPU power devices such as mobile.
             */
-            call_vpx!(vpx_codec_control_(&mut ctx, VP8E_SET_CPUUSED as _, speed,));
+            call_vpx!(vpx_codec_control_(&mut ctx, VP8E_SET_CPUUSED as _, 7,));
             // set row level multi-threading
             /*
             as some people in comments and below have already commented,
@@ -222,6 +211,19 @@ impl Encoder {
         })
     }
 
+    pub fn set_bitrate(&mut self, bitrate: c_uint) -> Result<()> {
+        // let mut cfg = self.ctx.config.enc;
+        let mut new_enc_cfg = unsafe { *self.ctx.config.enc.to_owned() };
+        new_enc_cfg.rc_target_bitrate = bitrate;
+        call_vpx!(vpx_codec_enc_config_set(&mut self.ctx, &new_enc_cfg));
+        return Ok(());
+    }
+
+    pub fn get_bitrate(&mut self) -> u32 {
+        let cfg = unsafe { *self.ctx.config.enc.to_owned() };
+        cfg.rc_target_bitrate
+    }
+
     /// Notify the encoder to return any pending packets
     pub fn flush(&mut self) -> Result<EncodeFrames> {
         call_vpx!(vpx_codec_encode(
@@ -273,9 +275,6 @@ pub struct Config {
     pub bitrate: c_uint,
     /// The codec
     pub codec: VideoCodecId,
-    pub rc_min_quantizer: u32,
-    pub rc_max_quantizer: u32,
-    pub speed: i32,
 }
 
 pub struct EncodeFrames<'a> {

libs/scrap/src/common/x11.rs

@@ -1,5 +1,5 @@
 use crate::x11;
-use std::{io, ops};
+use std::{io, ops, time::Duration};
 
 pub struct Capturer(x11::Capturer);
 
@@ -16,7 +16,7 @@ impl Capturer {
         self.0.display().rect().h as usize
     }
 
-    pub fn frame<'a>(&'a mut self, _timeout_ms: u32) -> io::Result<Frame<'a>> {
+    pub fn frame<'a>(&'a mut self, _timeout: Duration) -> io::Result<Frame<'a>> {
         Ok(Frame(self.0.frame()?))
     }
 }

src/client.rs

@@ -886,6 +886,8 @@ impl LoginConfigHandler {
             option.block_input = BoolOption::Yes.into();
         } else if name == "unblock-input" {
             option.block_input = BoolOption::No.into();
+        } else if name == "show-quality-monitor" {
+            config.show_quality_monitor = !config.show_quality_monitor;
         } else {
             let v = self.options.get(&name).is_some();
             if v {
@@ -918,15 +920,8 @@ impl LoginConfigHandler {
             n += 1;
         } else if q == "custom" {
             let config = PeerConfig::load(&self.id);
-            let mut it = config.custom_image_quality.iter();
+            msg.custom_image_quality = config.custom_image_quality[0] as _;
-            let bitrate = it.next();
+            n += 1;
-            let quantizer = it.next();
-            if let Some(bitrate) = bitrate {
-                if let Some(quantizer) = quantizer {
-                    msg.custom_image_quality = bitrate << 8 | quantizer;
-                    n += 1;
-                }
-            }
         }
         if self.get_toggle_option("show-remote-cursor") {
             msg.show_remote_cursor = BoolOption::Yes.into();
@@ -988,6 +983,8 @@ impl LoginConfigHandler {
             self.config.disable_audio
         } else if name == "disable-clipboard" {
             self.config.disable_clipboard
+        } else if name == "show-quality-monitor" {
+            self.config.show_quality_monitor
         } else {
             !self.get_option(name).is_empty()
         }
@@ -1009,17 +1006,17 @@ impl LoginConfigHandler {
         msg_out
     }
 
-    pub fn save_custom_image_quality(&mut self, bitrate: i32, quantizer: i32) -> Message {
+    pub fn save_custom_image_quality(&mut self, custom_image_quality: u32) -> Message {
         let mut misc = Misc::new();
         misc.set_option(OptionMessage {
-            custom_image_quality: bitrate << 8 | quantizer,
+            custom_image_quality,
             ..Default::default()
         });
         let mut msg_out = Message::new();
         msg_out.set_misc(misc);
         let mut config = self.load_config();
         config.image_quality = "custom".to_owned();
-        config.custom_image_quality = vec![bitrate, quantizer];
+        config.custom_image_quality = vec![custom_image_quality as _];
         self.save_config(config);
         msg_out
     }
@@ -1214,14 +1211,6 @@ where
     return (video_sender, audio_sender);
 }
 
-pub async fn handle_test_delay(t: TestDelay, peer: &mut Stream) {
-    if !t.from_client {
-        let mut msg_out = Message::new();
-        msg_out.set_test_delay(t);
-        allow_err!(peer.send(&msg_out).await);
-    }
-}
-
 // mask = buttons << 3 | type
 // type, 1: down, 2: up, 3: wheel
 // buttons, 1: left, 2: right, 4: middle

src/server/connection.rs

@@ -3,6 +3,7 @@ use super::{input_service::*, *};
 use crate::clipboard_file::*;
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
 use crate::common::update_clipboard;
+use crate::video_service;
 #[cfg(any(target_os = "android", target_os = "ios"))]
 use crate::{common::MOBILE_INFO2, mobile::connection_manager::start_channel};
 use crate::{ipc, VERSION};
@@ -69,7 +70,6 @@ pub struct Connection {
     audio: bool,
     file: bool,
     last_test_delay: i64,
-    image_quality: i32,
     lock_after_session_end: bool,
     show_remote_cursor: bool, // by peer
     ip: String,
@@ -105,7 +105,7 @@ impl Subscriber for ConnInner {
     }
 }
 
-const TEST_DELAY_TIMEOUT: Duration = Duration::from_secs(3);
+const TEST_DELAY_TIMEOUT: Duration = Duration::from_secs(1);
 const SEC30: Duration = Duration::from_secs(30);
 const H1: Duration = Duration::from_secs(3600);
 const MILLI1: Duration = Duration::from_millis(1);
@@ -154,7 +154,6 @@ impl Connection {
             audio: Config::get_option("enable-audio").is_empty(),
             file: Config::get_option("enable-file-transfer").is_empty(),
             last_test_delay: 0,
-            image_quality: ImageQuality::Balanced.value(),
             lock_after_session_end: false,
             show_remote_cursor: false,
             ip: "".to_owned(),
@@ -376,8 +375,11 @@ impl Connection {
                     if time > 0 && conn.last_test_delay == 0 {
                         conn.last_test_delay = time;
                         let mut msg_out = Message::new();
+                        let qos = video_service::VIDEO_QOS.lock().unwrap();
                         msg_out.set_test_delay(TestDelay{
                             time,
+                            last_delay:qos.current_delay,
+                            target_bitrate:qos.target_bitrate,
                             ..Default::default()
                         });
                         conn.inner.send(msg_out.into());
@@ -394,8 +396,7 @@ impl Connection {
             let _ = privacy_mode::turn_off_privacy(0);
         }
         video_service::notify_video_frame_feched(id, None);
-        video_service::update_test_latency(id, 0);
+        video_service::VIDEO_QOS.lock().unwrap().reset();
-        video_service::update_image_quality(id, None);
         if let Err(err) = conn.try_port_forward_loop(&mut rx_from_cm).await {
             conn.on_close(&err.to_string(), false);
         }
@@ -664,7 +665,7 @@ impl Connection {
             res.set_peer_info(pi);
         } else {
             try_activate_screen();
-            match super::video_service::get_displays() {
+            match video_service::get_displays() {
                 Err(err) => {
                     res.set_error(format!("X11 error: {}", err));
                 }
@@ -886,10 +887,11 @@ impl Connection {
                 self.inner.send(msg_out.into());
             } else {
                 self.last_test_delay = 0;
-                let latency = crate::get_time() - t.time;
+                let new_delay = (crate::get_time() - t.time) as u32;
-                if latency > 0 {
+                video_service::VIDEO_QOS
-                    super::video_service::update_test_latency(self.inner.id(), latency);
+                    .lock()
-                }
+                    .unwrap()
+                    .update_network_delay(new_delay);
             }
         } else if self.authorized {
             match msg.union {
@@ -1065,7 +1067,7 @@ impl Connection {
                 },
                 Some(message::Union::misc(misc)) => match misc.union {
                     Some(misc::Union::switch_display(s)) => {
-                        super::video_service::switch_display(s.display);
+                        video_service::switch_display(s.display);
                     }
                     Some(misc::Union::chat_message(c)) => {
                         self.send_to_cm(ipc::Data::ChatMessage { text: c.text });
@@ -1075,7 +1077,7 @@ impl Connection {
                     }
                     Some(misc::Union::refresh_video(r)) => {
                         if r {
-                            super::video_service::refresh();
+                            video_service::refresh();
                         }
                     }
                     Some(misc::Union::video_received(_)) => {
@@ -1095,13 +1097,18 @@ impl Connection {
     async fn update_option(&mut self, o: &OptionMessage) {
         log::info!("Option update: {:?}", o);
         if let Ok(q) = o.image_quality.enum_value() {
-            self.image_quality = q.value();
+            let mut image_quality = None;
-            super::video_service::update_image_quality(self.inner.id(), Some(q.value()));
+            if let ImageQuality::NotSet = q {
-        }
+                if o.custom_image_quality > 0 {
-        let q = o.custom_image_quality;
+                    image_quality = Some(o.custom_image_quality);
-        if q > 0 {
+                }
-            self.image_quality = q;
+            } else {
-            super::video_service::update_image_quality(self.inner.id(), Some(q));
+                image_quality = Some(q.value() as _)
+            }
+            video_service::VIDEO_QOS
+                .lock()
+                .unwrap()
+                .update_image_quality(image_quality);
         }
         if let Ok(q) = o.lock_after_session_end.enum_value() {
             if q != BoolOption::NotSet {

src/server/video_service.rs

@@ -33,19 +33,249 @@ use std::{
 use virtual_display;
 
 pub const NAME: &'static str = "video";
+const FPS: u8 = 30;
 
 lazy_static::lazy_static! {
     static ref CURRENT_DISPLAY: Arc<Mutex<usize>> = Arc::new(Mutex::new(usize::MAX));
     static ref LAST_ACTIVE: Arc<Mutex<Instant>> = Arc::new(Mutex::new(Instant::now()));
     static ref SWITCH: Arc<Mutex<bool>> = Default::default();
-    static ref TEST_LATENCIES: Arc<Mutex<HashMap<i32, i64>>> = Default::default();
-    static ref IMAGE_QUALITIES: Arc<Mutex<HashMap<i32, i32>>> = Default::default();
     static ref FRAME_FETCHED_NOTIFIER: (UnboundedSender<(i32, Option<Instant>)>, Arc<TokioMutex<UnboundedReceiver<(i32, Option<Instant>)>>>) = {
         let (tx, rx) = unbounded_channel();
         (tx, Arc::new(TokioMutex::new(rx)))
     };
     static ref PRIVACY_MODE_CONN_ID: Mutex<i32> = Mutex::new(0);
     static ref IS_CAPTURER_MAGNIFIER_SUPPORTED: bool = is_capturer_mag_supported();
+    pub static ref VIDEO_QOS: Arc<Mutex<VideoQoS>> = Default::default();
+}
+
+pub struct VideoQoS {
+    width: u32,
+    height: u32,
+    user_image_quality: u32,
+    current_image_quality: u32,
+    enable_abr: bool,
+
+    pub current_delay: u32,
+    pub fps: u8,             // abr
+    pub target_bitrate: u32, // abr
+    updated: bool,
+
+    state: AdaptiveState,
+    last_delay: u32,
+    count: u32,
+}
+
+#[derive(Debug)]
+enum AdaptiveState {
+    Normal,
+    LowDelay,
+    HeightDelay,
+}
+
+impl Default for VideoQoS {
+    fn default() -> Self {
+        VideoQoS {
+            fps: FPS,
+            user_image_quality: ImageQuality::Balanced.value() as _,
+            current_image_quality: ImageQuality::Balanced.value() as _,
+            enable_abr: false,
+            width: 0,
+            height: 0,
+            current_delay: 0,
+            target_bitrate: 0,
+            updated: false,
+            state: AdaptiveState::Normal,
+            last_delay: 0,
+            count: 0,
+        }
+    }
+}
+
+const MAX: f32 = 1.2;
+const MIN: f32 = 0.8;
+const MAX_COUNT: u32 = 3;
+const MAX_DELAY: u32 = 500;
+const MIN_DELAY: u32 = 50;
+
+impl VideoQoS {
+    pub fn set_size(&mut self, width: u32, height: u32) {
+        if width == 0 || height == 0 {
+            return;
+        }
+        self.width = width;
+        self.height = height;
+    }
+
+    pub fn spf(&mut self) -> Duration {
+        if self.fps <= 0 {
+            self.fps = FPS;
+        }
+        time::Duration::from_secs_f32(1. / (self.fps as f32))
+    }
+
+    // abr
+    pub fn update_network_delay(&mut self, delay: u32) {
+        if self.current_delay.eq(&0) {
+            self.current_delay = delay;
+            return;
+        }
+        let current_delay = self.current_delay as f32;
+
+        self.current_delay = delay / 2 + self.current_delay / 2;
+        log::debug!(
+            "update_network_delay:{}, {}, state:{:?},count:{}",
+            self.current_delay,
+            delay,
+            self.state,
+            self.count
+        );
+
+        if self.current_delay < MIN_DELAY {
+            if self.fps != 30 && self.current_image_quality != self.user_image_quality {
+                log::debug!("current_delay is normal, set to user_image_quality");
+                self.fps = 30;
+                self.current_image_quality = self.user_image_quality;
+                let _ = self.generate_bitrate().ok();
+                self.updated = true;
+            }
+            self.state = AdaptiveState::Normal;
+        } else if self.current_delay > MAX_DELAY {
+            if self.fps != 5 && self.current_image_quality != 25 {
+                log::debug!("current_delay is very height, set fps to 5, image_quality to 25");
+                self.fps = 5;
+                self.current_image_quality = 25;
+                let _ = self.generate_bitrate().ok();
+                self.updated = true;
+            }
+        } else {
+            let delay = delay as f32;
+            let last_delay = self.last_delay as f32;
+            match self.state {
+                AdaptiveState::Normal => {
+                    if delay > current_delay * MAX {
+                        self.state = AdaptiveState::HeightDelay;
+                    } else if delay < current_delay * MIN
+                        && self.current_image_quality < self.user_image_quality
+                    {
+                        self.state = AdaptiveState::LowDelay;
+                    }
+                    self.count = 1;
+                    self.last_delay = self.current_delay
+                }
+                AdaptiveState::HeightDelay => {
+                    if delay > last_delay {
+                        if self.count > MAX_COUNT {
+                            self.decrease_quality();
+                            self.reset_state();
+                            return;
+                        }
+                        self.count += 1;
+                    } else {
+                        self.reset_state();
+                    }
+                }
+                AdaptiveState::LowDelay => {
+                    if delay < last_delay * MIN {
+                        if self.count > MAX_COUNT {
+                            self.increase_quality();
+                            self.reset_state();
+                            return;
+                        }
+                        self.count += 1;
+                    } else {
+                        self.reset_state();
+                    }
+                }
+            }
+        }
+    }
+
+    fn reset_state(&mut self) {
+        self.count = 0;
+        self.state = AdaptiveState::Normal;
+    }
+
+    fn increase_quality(&mut self) {
+        log::debug!("Adaptive increase quality");
+        if self.fps < FPS {
+            log::debug!("increase fps {} -> {}", self.fps, FPS);
+            self.fps = FPS;
+        } else {
+            self.current_image_quality += self.current_image_quality / 2;
+            let _ = self.generate_bitrate().ok();
+            log::debug!("increase quality:{}", self.current_image_quality);
+        }
+        self.updated = true;
+    }
+
+    fn decrease_quality(&mut self) {
+        log::debug!("Adaptive decrease quality");
+        if self.fps < 15 {
+            log::debug!("fps is low enough :{}", self.fps);
+            return;
+        }
+        if self.current_image_quality < ImageQuality::Low.value() as _ {
+            self.fps = self.fps / 2;
+            log::debug!("decrease fps:{}", self.fps);
+        } else {
+            self.current_image_quality -= self.current_image_quality / 2;
+            let _ = self.generate_bitrate().ok();
+            log::debug!("decrease quality:{}", self.current_image_quality);
+        };
+        self.updated = true;
+    }
+
+    pub fn update_image_quality(&mut self, image_quality: Option<u32>) {
+        if let Some(image_quality) = image_quality {
+            if image_quality < 10 || image_quality > 200 {
+                self.current_image_quality = ImageQuality::Balanced.value() as _;
+            }
+            if self.current_image_quality != image_quality {
+                self.current_image_quality = image_quality;
+                let _ = self.generate_bitrate().ok();
+                self.updated = true;
+            }
+        } else {
+            self.current_image_quality = ImageQuality::Balanced.value() as _;
+        }
+        self.user_image_quality = self.current_image_quality;
+    }
+
+    pub fn generate_bitrate(&mut self) -> ResultType<u32> {
+        // https://www.nvidia.com/en-us/geforce/guides/broadcasting-guide/
+        if self.width == 0 || self.height == 0 {
+            bail!("Fail to generate_bitrate, width or height is not set");
+        }
+        if self.current_image_quality == 0 {
+            self.current_image_quality = ImageQuality::Balanced.value() as _;
+        }
+
+        let base_bitrate = ((self.width * self.height) / 800) as u32;
+
+        #[cfg(target_os = "android")]
+        {
+            // fix when andorid screen shrinks
+            let fix = Display::fix_quality() as u32;
+            log::debug!("Android screen, fix quality:{}", fix);
+            let base_bitrate = base_bitrate * fix;
+            self.target_bitrate = base_bitrate * self.image_quality / 100;
+            Ok(self.target_bitrate)
+        }
+        self.target_bitrate = base_bitrate * self.current_image_quality / 100;
+        Ok(self.target_bitrate)
+    }
+
+    pub fn check_if_updated(&mut self) -> bool {
+        if self.updated {
+            self.updated = false;
+            return true;
+        }
+        return false;
+    }
+
+    pub fn reset(&mut self) {
+        *self = Default::default();
+    }
 }
 
 fn is_capturer_mag_supported() -> bool {
@@ -125,7 +355,7 @@ impl VideoFrameController {
 }
 
 trait TraitCapturer {
-    fn frame<'a>(&'a mut self, timeout_ms: u32) -> Result<Frame<'a>>;
+    fn frame<'a>(&'a mut self, timeout: Duration) -> Result<Frame<'a>>;
 
     #[cfg(windows)]
     fn is_gdi(&self) -> bool;
@@ -134,8 +364,8 @@ trait TraitCapturer {
 }
 
 impl TraitCapturer for Capturer {
-    fn frame<'a>(&'a mut self, timeout_ms: u32) -> Result<Frame<'a>> {
+    fn frame<'a>(&'a mut self, timeout: Duration) -> Result<Frame<'a>> {
-        self.frame(timeout_ms)
+        self.frame(timeout)
     }
 
     #[cfg(windows)]
@@ -320,9 +550,6 @@ fn run(sp: GenericService) -> ResultType<()> {
     #[cfg(windows)]
     ensure_close_virtual_device()?;
 
-    let fps = 30;
-    let wait = 1000 / fps;
-    let spf = time::Duration::from_secs_f32(1. / (fps as f32));
     let (ndisplay, current, display) = get_current_display()?;
     let (origin, width, height) = (display.origin(), display.width(), display.height());
     log::debug!(
@@ -357,24 +584,26 @@ fn run(sp: GenericService) -> ResultType<()> {
     }
     let mut c = create_capturer(captuerer_privacy_mode_id, display)?;
 
-    let q = get_image_quality();
+    let mut video_qos = VIDEO_QOS.lock().unwrap();
-    let (bitrate, rc_min_quantizer, rc_max_quantizer, speed) = get_quality(width, height, q);
+
-    log::info!("bitrate={}, rc_min_quantizer={}", bitrate, rc_min_quantizer);
+    video_qos.set_size(width as _, height as _);
+    let mut spf = video_qos.spf();
+    let bitrate = video_qos.generate_bitrate()?;
+    drop(video_qos);
+
+    log::info!("init encoder, bitrate={}", bitrate);
     let cfg = Config {
         width: width as _,
         height: height as _,
         timebase: [1, 1000], // Output timestamp precision
         bitrate,
         codec: VideoCodecId::VP9,
-        rc_min_quantizer,
-        rc_max_quantizer,
-        speed,
     };
-    let mut vpx;
+
-    match Encoder::new(&cfg, (num_cpus::get() / 2) as _) {
+    let mut vpx = match Encoder::new(&cfg, (num_cpus::get() / 2) as _) {
-        Ok(x) => vpx = x,
+        Ok(x) => x,
         Err(err) => bail!("Failed to create encoder: {}", err),
-    }
+    };
 
     if *SWITCH.lock().unwrap() {
         log::debug!("Broadcasting display switch");
@@ -401,10 +630,24 @@ fn run(sp: GenericService) -> ResultType<()> {
     let mut try_gdi = 1;
     #[cfg(windows)]
     log::info!("gdi: {}", c.is_gdi());
+
     while sp.ok() {
         #[cfg(windows)]
         check_uac_switch(privacy_mode_id, captuerer_privacy_mode_id)?;
 
+        {
+            let mut video_qos = VIDEO_QOS.lock().unwrap();
+            if video_qos.check_if_updated() {
+                log::debug!(
+                    "qos is updated, target_bitrate:{}, fps:{}",
+                    video_qos.target_bitrate,
+                    video_qos.fps
+                );
+                vpx.set_bitrate(video_qos.target_bitrate).unwrap();
+                spf = video_qos.spf();
+            }
+        }
+
         if *SWITCH.lock().unwrap() {
             bail!("SWITCH");
         }
@@ -413,9 +656,6 @@ fn run(sp: GenericService) -> ResultType<()> {
             bail!("SWITCH");
         }
         check_privacy_mode_changed(&sp, privacy_mode_id)?;
-        if get_image_quality() != q {
-            bail!("SWITCH");
-        }
         #[cfg(windows)]
         {
             if crate::platform::windows::desktop_changed() {
@@ -437,7 +677,7 @@ fn run(sp: GenericService) -> ResultType<()> {
         frame_controller.reset();
 
         #[cfg(any(target_os = "android", target_os = "ios"))]
-        let res = match (*c).frame(wait as _) {
+        let res = match c.frame(spf) {
             Ok(frame) => {
                 let time = now - start;
                 let ms = (time.as_secs() * 1000 + time.subsec_millis() as u64) as i64;
@@ -460,7 +700,7 @@ fn run(sp: GenericService) -> ResultType<()> {
         };
 
         #[cfg(not(any(target_os = "android", target_os = "ios")))]
-        let res = match (*c).frame(wait as _) {
+        let res = match c.frame(spf) {
             Ok(frame) => {
                 let time = now - start;
                 let ms = (time.as_secs() * 1000 + time.subsec_millis() as u64) as i64;
@@ -748,82 +988,3 @@ fn get_current_display() -> ResultType<(usize, usize, Display)> {
     }
     return Ok((n, current, displays.remove(current)));
 }
-
-#[inline]
-fn update_latency(id: i32, latency: i64, latencies: &mut HashMap<i32, i64>) {
-    if latency <= 0 {
-        latencies.remove(&id);
-    } else {
-        latencies.insert(id, latency);
-    }
-}
-
-pub fn update_test_latency(id: i32, latency: i64) {
-    update_latency(id, latency, &mut *TEST_LATENCIES.lock().unwrap());
-}
-
-fn convert_quality(q: i32) -> i32 {
-    let q = {
-        if q == ImageQuality::Balanced.value() {
-            (100 * 2 / 3, 12)
-        } else if q == ImageQuality::Low.value() {
-            (100 / 2, 18)
-        } else if q == ImageQuality::Best.value() {
-            (100, 12)
-        } else {
-            let bitrate = q >> 8 & 0xFF;
-            let quantizer = q & 0xFF;
-            (bitrate * 2, (100 - quantizer) * 36 / 100)
-        }
-    };
-    if q.0 <= 0 {
-        0
-    } else {
-        q.0 << 8 | q.1
-    }
-}
-
-pub fn update_image_quality(id: i32, q: Option<i32>) {
-    match q {
-        Some(q) => {
-            let q = convert_quality(q);
-            if q > 0 {
-                IMAGE_QUALITIES.lock().unwrap().insert(id, q);
-            } else {
-                IMAGE_QUALITIES.lock().unwrap().remove(&id);
-            }
-        }
-        None => {
-            IMAGE_QUALITIES.lock().unwrap().remove(&id);
-        }
-    }
-}
-
-fn get_image_quality() -> i32 {
-    IMAGE_QUALITIES
-        .lock()
-        .unwrap()
-        .values()
-        .min()
-        .unwrap_or(&convert_quality(ImageQuality::Balanced.value()))
-        .clone()
-}
-
-#[inline]
-fn get_quality(w: usize, h: usize, q: i32) -> (u32, u32, u32, i32) {
-    // https://www.nvidia.com/en-us/geforce/guides/broadcasting-guide/
-    let bitrate = q >> 8 & 0xFF;
-    let quantizer = q & 0xFF;
-    let b = ((w * h) / 1000) as u32;
-
-    #[cfg(target_os = "android")]
-    {
-        // fix when andorid screen shrinks
-        let fix = Display::fix_quality() as u32;
-        log::debug!("Android screen, fix quality:{}", fix);
-        let b = b * fix;
-        return (bitrate as u32 * b / 100, quantizer as _, 56, 7);
-    }
-
-    (bitrate as u32 * b / 100, quantizer as _, 56, 7)
-}

src/ui/header.tis

@@ -159,6 +159,7 @@ class Header: Reactor.Component {
                 <li #custom type="image-quality"><span>{svg_checkmark}</span>{translate('Custom')}</li>
                 <div .separator />
                 <li #show-remote-cursor .toggle-option><span>{svg_checkmark}</span>{translate('Show remote cursor')}</li> 
+                <li #show-quality-monitor .toggle-option><span>{svg_checkmark}</span>{translate('Show quality monitor')}</li> 
                 {audio_enabled ? <li #disable-audio .toggle-option><span>{svg_checkmark}</span>{translate('Mute')}</li> : ""}
                 {is_win && pi.platform == 'Windows' && file_enabled ? <li #enable-file-transfer .toggle-option><span>{svg_checkmark}</span>{translate('Allow file copy and paste')}</li> : ""}
                 {keyboard_enabled && clipboard_enabled ? <li #disable-clipboard .toggle-option><span>{svg_checkmark}</span>{translate('Disable clipboard')}</li> : ""} 
@@ -315,7 +316,9 @@ class Header: Reactor.Component {
             handle_custom_image_quality();
         } else if (me.id == "privacy-mode") {
             togglePrivacyMode(me.id);
-        } else if (me.attributes.hasClass("toggle-option")) {
+        } else if (me.id == "show-quality-monitor") {
+            toggleQualityMonitor(me.id);
+        }else if (me.attributes.hasClass("toggle-option")) {
             handler.toggle_option(me.id);
             toggleMenuState();
         } else if (!me.attributes.hasClass("selected")) {
@@ -332,16 +335,13 @@ class Header: Reactor.Component {
 }
 
 function handle_custom_image_quality() {
-    var tmp = handler.get_custom_image_quality();
+    var bitrate = handler.get_custom_image_quality()[0] / 2;
-    var bitrate0 = tmp[0] || 50;
-    var quantizer0 = tmp.length > 1 ? tmp[1] : 100;
     msgbox("custom", "Custom Image Quality", "<div .form> \
-          <div><input type=\"hslider\" style=\"width: 50%\" name=\"bitrate\" max=\"100\" min=\"10\" value=\"" + bitrate0 + "\"/ buddy=\"bitrate-buddy\"><b #bitrate-buddy>x</b>% bitrate</div> \
+          <div><input type=\"hslider\" style=\"width: 50%\" name=\"bitrate\" max=\"100\" min=\"10\" value=\"" + bitrate + "\"/ buddy=\"bitrate-buddy\"><b #bitrate-buddy>x</b>% Bitrate</div> \
-          <div><input type=\"hslider\" style=\"width: 50%\" name=\"quantizer\" max=\"100\" min=\"0\" value=\"" + quantizer0 + "\"/ buddy=\"quantizer-buddy\"><b #quantizer-buddy>x</b>% quantizer</div> \
       </div>", function(res=null) {
         if (!res) return;
         if (!res.bitrate) return;
-        handler.save_custom_image_quality(res.bitrate, res.quantizer);
+        handler.save_custom_image_quality(res.bitrate * 2);
         toggleMenuState();
       });
 }
@@ -357,7 +357,7 @@ function toggleMenuState() {
     for (var el in $$(menu#display-options>li)) {
         el.attributes.toggleClass("selected", values.indexOf(el.id) >= 0);
     }
-    for (var id in ["show-remote-cursor", "disable-audio", "enable-file-transfer", "disable-clipboard", "lock-after-session-end"]) {
+    for (var id in ["show-remote-cursor", "show-quality-monitor", "disable-audio", "enable-file-transfer", "disable-clipboard", "lock-after-session-end"]) {
         var el = self.select('#' + id);
         if (el) {
             var value = handler.get_toggle_option(id);
@@ -425,6 +425,17 @@ function togglePrivacyMode(privacy_id) {
     }
 }
 
+function toggleQualityMonitor(name) {
+    var show = handler.get_toggle_option(name);
+    if (show) {
+        $(#quality-monitor).style.set{ display: "none" };
+    } else {
+        $(#quality-monitor).style.set{ display: "block" };
+    }
+    handler.toggle_option(name);
+    toggleMenuState();
+}
+
 handler.updateBlockInputState = function(input_blocked) {
     if (!input_blocked) {
         handler.toggle_option("block-input");

src/ui/remote.css

@@ -9,6 +9,16 @@ div#video-wrapper {
     background: #212121;
 }
 
+div#quality-monitor {
+    top: 20px;
+    right: 20px;
+    background: #7571719c;
+    padding: 5px;
+    min-width: 150px;
+    color: azure;
+    border: solid azure;
+}
+
 video#handler {
     behavior: native-remote video;
     size: *;
@@ -24,7 +34,7 @@ img#cursor {
 }
 
 .goup {
-  transform: rotate(90deg);
+    transform: rotate(90deg);
 }
 
 table#remote-folder-view {
@@ -33,4 +43,4 @@ table#remote-folder-view {
 
 table#local-folder-view {
     context-menu: selector(menu#local-folder-view);
-}
+}

src/ui/remote.html

@@ -1,12 +1,13 @@
 <html window-resizable window-frame="extended">
-    <head>
+
-        <style>
+<head>
-            @import url(common.css);
+    <style>
-            @import url(remote.css);
+        @import url(common.css);
-            @import url(file_transfer.css);
+        @import url(remote.css);
-            @import url(header.css);
+        @import url(file_transfer.css);
-        </style>
+        @import url(header.css);
-        <script type="text/tiscript">
+    </style>
+    <script type="text/tiscript">
             include "common.tis";
             include "msgbox.tis";
             include "remote.tis";
@@ -15,23 +16,28 @@
             include "grid.tis";
             include "header.tis";
         </script>
-    </head>
+</head>
-    <header>
+<header>
-        <div.window-icon role="window-icon"><icon /></div>
+    <div.window-icon role="window-icon">
+        <icon />
+        </div>
         <caption role="window-caption" />
         <div.window-toolbar />
         <div.window-buttons />
-    </header>
+</header>
-    <body>
+
-        <div #video-wrapper>
+<body>
-            <video #handler>
+    <div #video-wrapper>
-                <div style="position: relative">
+        <video #handler>
-                    <img #cursor src="in-memory:cursor" />
+            <div #quality-monitor style="position: absolute; display: none" />
-                </div>
+            <div style="position: relative">
-            </video>
+                <img #cursor src="in-memory:cursor" />
-        </div>
+            </div>
-        <div #file-transfer-wrapper>
+        </video>
-        </div>
+    </div>
-        <div #msgbox />
+    <div #file-transfer-wrapper>
-    </body>
+    </div>
-</html>
+    <div #msgbox />
+</body>
+
+</html>

src/ui/remote.rs

@@ -2,7 +2,7 @@ use std::{
     collections::HashMap,
     ops::Deref,
     sync::{
-        atomic::{AtomicBool, Ordering},
+        atomic::{AtomicBool, AtomicUsize, Ordering},
         Arc, Mutex, RwLock,
     },
 };
@@ -223,7 +223,7 @@ impl sciter::EventHandler for Handler {
         fn get_custom_image_quality();
         fn save_view_style(String);
         fn save_image_quality(String);
-        fn save_custom_image_quality(i32, i32);
+        fn save_custom_image_quality(i32);
         fn refresh_video();
         fn get_toggle_option(String);
         fn is_privacy_mode_supported();
@@ -234,6 +234,25 @@ impl sciter::EventHandler for Handler {
     }
 }
 
+#[derive(Debug)]
+struct QualityStatus {
+    speed: String,
+    fps: i32,
+    delay: i32,
+    target_bitrate: i32,
+}
+
+impl Default for QualityStatus {
+    fn default() -> Self {
+        Self {
+            speed: Default::default(),
+            fps: -1,
+            delay: -1,
+            target_bitrate: -1,
+        }
+    }
+}
+
 impl Handler {
     pub fn new(cmd: String, id: String, args: Vec<String>) -> Self {
         let me = Self {
@@ -249,6 +268,18 @@ impl Handler {
         me
     }
 
+    fn update_quality_status(&self, status: QualityStatus) {
+        self.call2(
+            "updateQualityStatus",
+            &make_args!(
+                status.speed,
+                status.fps,
+                status.delay,
+                status.target_bitrate
+            ),
+        );
+    }
+
     fn start_keyboard_hook(&self) {
         if self.is_port_forward() || self.is_file_transfer() {
             return;
@@ -533,12 +564,12 @@ impl Handler {
         self.send(Data::Message(LoginConfigHandler::refresh()));
     }
 
-    fn save_custom_image_quality(&mut self, bitrate: i32, quantizer: i32) {
+    fn save_custom_image_quality(&mut self, custom_image_quality: i32) {
         let msg = self
             .lc
             .write()
             .unwrap()
-            .save_custom_image_quality(bitrate, quantizer);
+            .save_custom_image_quality(custom_image_quality as u32);
         self.send(Data::Message(msg));
     }
 
@@ -1296,7 +1327,10 @@ async fn io_loop(handler: Handler) {
         }
         return;
     }
-    let (video_sender, audio_sender) = start_video_audio_threads(|data: &[u8]| {
+    let frame_count = Arc::new(AtomicUsize::new(0));
+    let frame_count_cl = frame_count.clone();
+    let (video_sender, audio_sender) = start_video_audio_threads(move |data: &[u8]| {
+        frame_count_cl.fetch_add(1, Ordering::Relaxed);
         VIDEO
             .lock()
             .unwrap()
@@ -1319,6 +1353,8 @@ async fn io_loop(handler: Handler) {
         first_frame: false,
         #[cfg(windows)]
         clipboard_file_context: None,
+        data_count: Arc::new(AtomicUsize::new(0)),
+        frame_count,
     };
     remote.io_loop(&key, &token).await;
     remote.sync_jobs_status_to_local().await;
@@ -1369,6 +1405,8 @@ struct Remote {
     first_frame: bool,
     #[cfg(windows)]
     clipboard_file_context: Option<Box<CliprdrClientContext>>,
+    data_count: Arc<AtomicUsize>,
+    frame_count: Arc<AtomicUsize>,
 }
 
 impl Remote {
@@ -1394,6 +1432,8 @@ impl Remote {
                 #[cfg(windows)]
                 let mut rx_clip_client = get_rx_clip_client().lock().await;
 
+                let mut status_timer = time::interval(Duration::new(1, 0));
+
                 loop {
                     tokio::select! {
                         res = peer.next() => {
@@ -1406,6 +1446,7 @@ impl Remote {
                                     }
                                     Ok(ref bytes) => {
                                         last_recv_time = Instant::now();
+                                        self.data_count.fetch_add(bytes.len(), Ordering::Relaxed);
                                         if !self.handle_msg_from_peer(bytes, &mut peer).await {
                                             break
                                         }
@@ -1450,6 +1491,16 @@ impl Remote {
                                 self.timer = time::interval_at(Instant::now() + SEC30, SEC30);
                             }
                         }
+                        _ = status_timer.tick() => {
+                            let speed = self.data_count.swap(0, Ordering::Relaxed);
+                            let speed = format!("{:.2}kB/s", speed as f32 / 1024 as f32);
+                            let fps = self.frame_count.swap(0, Ordering::Relaxed) as _;
+                            self.handler.update_quality_status(QualityStatus {
+                                speed,
+                                fps,
+                                ..Default::default()
+                            });
+                        }
                     }
                 }
                 log::debug!("Exit io_loop of id={}", self.handler.id);
@@ -2370,7 +2421,7 @@ impl Remote {
             }
             back_notification::PrivacyModeState::OffSucceeded => {
                 self.handler
-                .msgbox("custom-nocancel", "Privacy mode", "Out privacy mode");
+                    .msgbox("custom-nocancel", "Privacy mode", "Out privacy mode");
                 self.update_privacy_mode(false);
             }
             back_notification::PrivacyModeState::OffByPeer => {
@@ -2549,7 +2600,16 @@ impl Interface for Handler {
     }
 
     async fn handle_test_delay(&mut self, t: TestDelay, peer: &mut Stream) {
-        handle_test_delay(t, peer).await;
+        if !t.from_client {
+            self.update_quality_status(QualityStatus {
+                delay: t.last_delay as _,
+                target_bitrate: t.target_bitrate as _,
+                ..Default::default()
+            });
+            let mut msg_out = Message::new();
+            msg_out.set_test_delay(t);
+            allow_err!(peer.send(&msg_out).await);
+        }
     }
 }
 

src/ui/remote.tis

@@ -456,6 +456,45 @@ function self.closing() {
     if (is_file_transfer || is_port_forward || size_adapted) handler.save_size(x, y, w, h);
 }
 
+var qualityMonitor;
+var qualityMonitorData = [];
+
+class QualityMonitor: Reactor.Component
+{
+    function this() {
+        qualityMonitor = this;
+        if (handler.get_toggle_option("show-quality-monitor")) {
+            $(#quality-monitor).style.set{ display: "block" };
+        }
+    }
+
+    function render() {
+        return <div >
+            <div>
+                Speed: {qualityMonitorData[0]}
+            </div>
+            <div>
+                FPS: {qualityMonitorData[1]}
+            </div>
+            <div>
+                Delay: {qualityMonitorData[2]} ms
+            </div>
+            <div>
+                Target Bitrate: {qualityMonitorData[3]}kb
+            </div>
+        </div>;
+    }
+}
+
+$(#quality-monitor).content(<QualityMonitor />);
+handler.updateQualityStatus = function(speed, fps, delay, bitrate) {
+    speed ? qualityMonitorData[0] = speed:null;
+    fps > -1 ? qualityMonitorData[1] = fps:null;
+    delay > -1 ? qualityMonitorData[2] = delay:null;
+    bitrate > -1 ? qualityMonitorData[3] = bitrate:null;
+    qualityMonitor.update();
+}
+
 handler.setPermission = function(name, enabled) {
     self.timer(60ms, function() {
     if (name == "keyboard") keyboard_enabled = enabled;