videotoolbox ram codec (#8683)

* h265 encoding: the second frame always fails, use repeat encoding to achieve real-time encoding
* h264 encoding: Not supported because encoding fails too frequently, about 50%, with one failure followed by another success.
* h264/h265 decoding: No issues found.
* Does not support dynamically changing the bitrate and changing the quality by resetting the encoder.

Signed-off-by: 21pages <sunboeasy@gmail.com>

Cargo.lock

@@ -3087,8 +3087,8 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
 [[package]]
 name = "hwcodec"
-version = "0.4.18"
-source = "git+https://github.com/21pages/hwcodec#4b15d782512f95cb158577853e6cdb67a37502c1"
+version = "0.4.19"
+source = "git+https://github.com/21pages/hwcodec#852de98224605db7a2495cf276776dabaf8aa139"
 dependencies = [
  "bindgen 0.59.2",
  "cc",

libs/scrap/src/common/codec.rs

@@ -838,7 +838,7 @@ impl Decoder {
 pub fn enable_hwcodec_option() -> bool {
     use hbb_common::config::keys::OPTION_ENABLE_HWCODEC;
 
-    if cfg!(windows) || cfg!(target_os = "linux") || cfg!(target_os = "android") {
+    if !cfg!(target_os = "ios") {
         return option2bool(
             OPTION_ENABLE_HWCODEC,
             &Config::get_option(OPTION_ENABLE_HWCODEC),

libs/scrap/src/common/hwcodec.rs

@@ -192,19 +192,21 @@ impl EncoderApi for HwRamEncoder {
     }
 
     fn support_abr(&self) -> bool {
-        ["qsv", "vaapi", "mediacodec"]
+        ["qsv", "vaapi", "mediacodec", "videotoolbox"]
             .iter()
             .all(|&x| !self.config.name.contains(x))
     }
 
     fn support_changing_quality(&self) -> bool {
-        ["vaapi", "mediacodec"]
+        ["vaapi", "mediacodec", "videotoolbox"]
             .iter()
             .all(|&x| !self.config.name.contains(x))
     }
 
     fn latency_free(&self) -> bool {
-        !self.config.name.contains("mediacodec")
+        ["mediacodec", "videotoolbox"]
+            .iter()
+            .all(|&x| !self.config.name.contains(x))
     }
 
     fn is_hardware(&self) -> bool {
@@ -501,12 +503,12 @@ pub struct HwCodecConfig {
 // portable: ui start check process, check process send to ui
 // sciter and unilink: get from ipc server
 impl HwCodecConfig {
-    #[cfg(any(target_os = "windows", target_os = "linux"))]
+    #[cfg(not(any(target_os = "android", target_os = "ios")))]
     pub fn set(config: String) {
         let config = serde_json::from_str(&config).unwrap_or_default();
         log::info!("set hwcodec config");
         log::debug!("{config:?}");
-        #[cfg(windows)]
+        #[cfg(any(windows, target_os = "macos"))]
         hbb_common::config::common_store(&config, "_hwcodec");
         *CONFIG.lock().unwrap() = Some(config);
         *CONFIG_SET_BY_IPC.lock().unwrap() = true;
@@ -578,7 +580,7 @@ impl HwCodecConfig {
                 ..Default::default()
             }
         }
-        #[cfg(windows)]
+        #[cfg(any(windows, target_os = "macos"))]
         {
             let config = CONFIG.lock().unwrap().clone();
             match config {
@@ -606,13 +608,13 @@ impl HwCodecConfig {
         {
             CONFIG.lock().unwrap().clone().unwrap_or_default()
         }
-        #[cfg(any(target_os = "macos", target_os = "ios"))]
+        #[cfg(target_os = "ios")]
         {
             HwCodecConfig::default()
         }
     }
 
-    #[cfg(any(target_os = "windows", target_os = "linux"))]
+    #[cfg(not(any(target_os = "android", target_os = "ios")))]
     pub fn get_set_value() -> Option<HwCodecConfig> {
         let set = CONFIG_SET_BY_IPC.lock().unwrap().clone();
         if set {
@@ -622,7 +624,7 @@ impl HwCodecConfig {
         }
     }
 
-    #[cfg(any(target_os = "windows", target_os = "linux"))]
+    #[cfg(not(any(target_os = "android", target_os = "ios")))]
     pub fn already_set() -> bool {
         CONFIG_SET_BY_IPC.lock().unwrap().clone()
     }
@@ -690,7 +692,7 @@ pub fn check_available_hwcodec() -> String {
     serde_json::to_string(&c).unwrap_or_default()
 }
 
-#[cfg(any(target_os = "windows", target_os = "linux"))]
+#[cfg(not(any(target_os = "android", target_os = "ios")))]
 pub fn start_check_process() {
     if !enable_hwcodec_option() || HwCodecConfig::already_set() {
         return;

src/ipc.rs

@@ -553,12 +553,12 @@ async fn handle(data: Data, stream: &mut Connection) {
             );
         }
         #[cfg(feature = "hwcodec")]
-        #[cfg(any(target_os = "windows", target_os = "linux"))]
+        #[cfg(not(any(target_os = "android", target_os = "ios")))]
         Data::CheckHwcodec => {
             scrap::hwcodec::start_check_process();
         }
         #[cfg(feature = "hwcodec")]
-        #[cfg(any(target_os = "windows", target_os = "linux"))]
+        #[cfg(not(any(target_os = "android", target_os = "ios")))]
         Data::HwCodecConfig(c) => {
             match c {
                 None => {
@@ -1047,7 +1047,7 @@ pub async fn notify_server_to_check_hwcodec() -> ResultType<()> {
 }
 
 #[cfg(feature = "hwcodec")]
-#[cfg(any(target_os = "windows", target_os = "linux"))]
+#[cfg(not(any(target_os = "android", target_os = "ios")))]
 #[tokio::main(flavor = "current_thread")]
 pub async fn get_hwcodec_config_from_server() -> ResultType<()> {
     if !scrap::codec::enable_hwcodec_option() || scrap::hwcodec::HwCodecConfig::already_set() {
@@ -1070,7 +1070,7 @@ pub async fn get_hwcodec_config_from_server() -> ResultType<()> {
 }
 
 #[cfg(feature = "hwcodec")]
-#[cfg(any(target_os = "windows", target_os = "linux"))]
+#[cfg(not(any(target_os = "android", target_os = "ios")))]
 pub fn client_get_hwcodec_config_thread(wait_sec: u64) {
     static ONCE: std::sync::Once = std::sync::Once::new();
     if !crate::platform::is_installed()

src/server.rs

@@ -489,7 +489,7 @@ pub async fn start_server(is_server: bool) {
         #[cfg(target_os = "windows")]
         crate::platform::try_kill_broker();
         #[cfg(feature = "hwcodec")]
-        #[cfg(any(target_os = "windows", target_os = "linux"))]
+        #[cfg(not(any(target_os = "android", target_os = "ios")))]
         scrap::hwcodec::start_check_process();
         crate::RendezvousMediator::start_all().await;
     } else {

src/ui_interface.rs

@@ -904,8 +904,7 @@ pub fn get_api_server() -> String {
 #[inline]
 pub fn has_hwcodec() -> bool {
     // Has real hardware codec using gpu
-    (cfg!(feature = "hwcodec") && (cfg!(windows) || cfg!(target_os = "linux")))
-        || cfg!(feature = "mediacodec")
+    (cfg!(feature = "hwcodec") && cfg!(not(target_os = "ios"))) || cfg!(feature = "mediacodec")
 }
 
 #[inline]
@@ -1408,7 +1407,7 @@ pub fn verify_bot(token: String) -> String {
 
 pub fn check_hwcodec() {
     #[cfg(feature = "hwcodec")]
-    #[cfg(any(target_os = "windows", target_os = "linux"))]
+    #[cfg(not(any(target_os = "android", target_os = "ios")))]
     {
         use std::sync::Once;
         static ONCE: Once = Once::new();