enable ffmpeg native h26x software decoders for all platforms (#7750)

* enable ffmpeg native h26x software decoders for all platforms

* h26x software decoders depend on hwcodec feature, so all platforms
  enable it, software h26x decoders are always available like vpx, no available check and no option
* ffmpeg:
	- build: mac arm64 build ffmpeg with my m1, others build with ci
	- version: win/linux use ffmpeg release/5.1, becaues higher version require higher nvidia driver,  other platforms use release/7.0
* test:
	- ios not test.
	- android: sometimes the screen will appear blurry, but it will recover after a while.
	- arm64 linux: test a example of hwcodec repo

Signed-off-by: 21pages <pages21@163.com>

* check hwcodec only when enabled and immediately when clicked enabled

Signed-off-by: 21pages <pages21@163.com>

---------

Signed-off-by: 21pages <pages21@163.com>

.github/workflows/flutter-build.yml

@@ -331,8 +331,7 @@ jobs:
 
       - name: Build rustdesk
         run: |
-          # --hwcodec not supported on macos yet
+          ./build.py --flutter --hwcodec
-          ./build.py --flutter
 
       - name: create unsigned dmg
         if: env.UPLOAD_ARTIFACT == 'true'
@@ -486,8 +485,7 @@ jobs:
 
       - name: Build rustdesk
         run: |
-          # --hwcodec not supported on macos yet
+          ./build.py --flutter --hwcodec ${{ matrix.job.extra-build-args }}
-          ./build.py --flutter ${{ matrix.job.extra-build-args }}
 
       - name: create unsigned dmg
         if: env.UPLOAD_ARTIFACT == 'true'
@@ -636,7 +634,7 @@ jobs:
       - name: Build rustdesk lib
         run: |
           rustup target add ${{ matrix.job.target }}
-          cargo build --features flutter --release --target aarch64-apple-ios --lib
+          cargo build --features flutter,hwcodec --release --target aarch64-apple-ios --lib
 
       - name: Build rustdesk
         shell: bash
@@ -1257,7 +1255,7 @@ jobs:
               export DEFAULT_FEAT=linux_headless
             fi
             export CARGO_INCREMENTAL=0
-            cargo build --lib --features flutter,flutter_texture_render,${{ matrix.job.extra-build-features }},$DEFAULT_FEAT --release
+            cargo build --lib --features flutter,flutter_texture_render,hwcodec,${{ matrix.job.extra-build-features }},$DEFAULT_FEAT --release
 
       - name: Upload Artifacts
         uses: actions/upload-artifact@master
@@ -1432,7 +1430,7 @@ jobs:
             if ${{ matrix.job.enable-headless }}; then
               export DEFAULT_FEAT=linux_headless
             fi
-            cargo build --features inline,${{ matrix.job.extra-build-features }},$DEFAULT_FEAT --release --bins
+            cargo build --features inline,hwcodec,${{ matrix.job.extra-build-features }},$DEFAULT_FEAT --release --bins
             # package
             mkdir -p ./Release
             mv ./target/release/rustdesk ./Release/rustdesk

Cargo.lock

@@ -3024,8 +3024,8 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
 [[package]]
 name = "hwcodec"
-version = "0.3.0"
+version = "0.3.2"
-source = "git+https://github.com/21pages/hwcodec#6ce1cbab2ff270a81784303192e8906ef597ee02"
+source = "git+https://github.com/21pages/hwcodec#1b754302d884d6d385a8f775acc514248006e3bb"
 dependencies = [
  "bindgen 0.59.2",
  "cc",

flutter/lib/desktop/pages/desktop_setting_page.dart

@@ -404,7 +404,16 @@ class _GeneralState extends State<_General> {
     return Offstage(
       offstage: !(hwcodec || vram),
       child: _Card(title: 'Hardware Codec', children: [
-        _OptionCheckBox(context, 'Enable hardware codec', 'enable-hwcodec')
+        _OptionCheckBox(
+          context,
+          'Enable hardware codec',
+          'enable-hwcodec',
+          update: () {
+            if (mainGetBoolOptionSync('enable-hwcodec')) {
+              bind.mainCheckHwcodec();
+            }
+          },
+        )
       ]),
     );
   }

flutter/lib/web/bridge.dart

@@ -1573,5 +1573,9 @@ class RustdeskImpl {
     throw UnimplementedError();
   }
 
+  Future<void> mainCheckHwcodec({dynamic hint}) {
+    throw UnimplementedError();
+  }
+
   void dispose() {}
 }

flutter/ndk_arm.sh

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-cargo ndk --platform 21 --target armv7-linux-androideabi build --release --features flutter
+cargo ndk --platform 21 --target armv7-linux-androideabi build --release --features flutter,hwcodec

flutter/ndk_arm64.sh

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-cargo ndk --platform 21 --target aarch64-linux-android build --release --features flutter
+cargo ndk --platform 21 --target aarch64-linux-android build --release --features flutter,hwcodec

libs/scrap/Cargo.toml

@@ -61,6 +61,5 @@ gstreamer-video = { version = "0.16", optional = true }
 [dependencies.hwcodec]
 git = "https://github.com/21pages/hwcodec"
 optional = true
-features = ["ffmpeg"]
 
 

libs/scrap/src/common/codec.rs

@@ -399,7 +399,7 @@ impl Decoder {
             ..Default::default()
         };
         #[cfg(feature = "hwcodec")]
-        if enable_hwcodec_option() {
+        {
             let best = HwRamDecoder::best();
             decoding.ability_h264 |= if best.h264.is_some() { 1 } else { 0 };
             decoding.ability_h265 |= if best.h265.is_some() { 1 } else { 0 };
@@ -492,7 +492,7 @@ impl Decoder {
                     valid = h264_vram.is_some();
                 }
                 #[cfg(feature = "hwcodec")]
-                if !valid && enable_hwcodec_option() {
+                if !valid {
                     match HwRamDecoder::new(format) {
                         Ok(v) => h264_ram = Some(v),
                         Err(e) => log::error!("create H264 ram decoder failed: {}", e),
@@ -518,7 +518,7 @@ impl Decoder {
                     valid = h265_vram.is_some();
                 }
                 #[cfg(feature = "hwcodec")]
-                if !valid && enable_hwcodec_option() {
+                if !valid {
                     match HwRamDecoder::new(format) {
                         Ok(v) => h265_ram = Some(v),
                         Err(e) => log::error!("create H265 ram decoder failed: {}", e),
@@ -792,10 +792,13 @@ impl Decoder {
 
 #[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
 pub fn enable_hwcodec_option() -> bool {
-    if let Some(v) = Config2::get().options.get("enable-hwcodec") {
+    if cfg!(windows) || cfg!(target_os = "linux") || cfg!(feature = "mediacodec") {
-        return v != "N";
+        if let Some(v) = Config2::get().options.get("enable-hwcodec") {
+            return v != "N";
+        }
+        return true; // default is true
     }
-    return true; // default is true
+    false
 }
 #[cfg(feature = "vram")]
 pub fn enable_vram_option() -> bool {

libs/scrap/src/common/hwcodec.rs

@@ -1,9 +1,10 @@
 use crate::{
-    codec::{base_bitrate, codec_thread_num, EncoderApi, EncoderCfg, Quality as Q},
+    codec::{
+        base_bitrate, codec_thread_num, enable_hwcodec_option, EncoderApi, EncoderCfg, Quality as Q,
+    },
     hw, CodecFormat, EncodeInput, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN,
 };
 use hbb_common::{
-    allow_err,
     anyhow::{anyhow, bail, Context},
     bytes::Bytes,
     config::HwCodecConfig,
@@ -223,10 +224,18 @@ pub struct HwRamDecoder {
 
 impl HwRamDecoder {
     pub fn best() -> CodecInfos {
-        get_config().map(|c| c.d).unwrap_or(CodecInfos {
+        let mut info = CodecInfo::soft();
-            h264: None,
+        if enable_hwcodec_option() {
-            h265: None,
+            if let Ok(hw) = get_config().map(|c| c.d) {
-        })
+                if let Some(h264) = hw.h264 {
+                    info.h264 = Some(h264);
+                }
+                if let Some(h265) = hw.h265 {
+                    info.h265 = Some(h265);
+                }
+            }
+        }
+        info
     }
 
     pub fn new(format: CodecFormat) -> ResultType<Self> {
@@ -359,8 +368,8 @@ pub fn check_available_hwcodec() {
     let vram = crate::vram::check_available_vram();
     #[cfg(not(feature = "vram"))]
     let vram = "".to_owned();
-    let encoders = CodecInfo::score(Encoder::available_encoders(ctx, Some(vram.clone())));
+    let encoders = CodecInfo::prioritized(Encoder::available_encoders(ctx, Some(vram.clone())));
-    let decoders = CodecInfo::score(Decoder::available_decoders(Some(vram.clone())));
+    let decoders = CodecInfo::prioritized(Decoder::available_decoders(Some(vram.clone())));
     let ram = Available {
         e: encoders,
         d: decoders,
@@ -370,7 +379,12 @@ pub fn check_available_hwcodec() {
     }
 }
 
-pub fn hwcodec_new_check_process() {
+#[cfg(any(target_os = "windows", target_os = "linux"))]
+pub fn start_check_process(force: bool) {
+    if !force && !enable_hwcodec_option() {
+        return;
+    }
+    use hbb_common::allow_err;
     use std::sync::Once;
     let f = || {
         // Clear to avoid checking process errors
@@ -410,7 +424,11 @@ pub fn hwcodec_new_check_process() {
         };
     };
     static ONCE: Once = Once::new();
-    ONCE.call_once(|| {
+    if force && ONCE.is_completed() {
         std::thread::spawn(f);
-    });
+    } else {
+        ONCE.call_once(|| {
+            std::thread::spawn(f);
+        });
+    }
 }

src/flutter_ffi.rs

@@ -2100,6 +2100,10 @@ pub fn main_get_hard_option(key: String) -> SyncReturn<String> {
     SyncReturn(get_hard_option(key))
 }
 
+pub fn main_check_hwcodec() {
+    check_hwcodec()
+}
+
 #[cfg(target_os = "android")]
 pub mod server_side {
     use hbb_common::{config, log};

src/ipc.rs

@@ -232,6 +232,7 @@ pub enum Data {
     #[cfg(windows)]
     ControlledSessionCount(usize),
     CmErr(String),
+    CheckHwcodec,
 }
 
 #[tokio::main(flavor = "current_thread")]
@@ -502,6 +503,14 @@ async fn handle(data: Data, stream: &mut Connection) {
                     .await
             );
         }
+        Data::CheckHwcodec =>
+        {
+            #[cfg(feature = "hwcodec")]
+            #[cfg(any(target_os = "windows", target_os = "linux"))]
+            if crate::platform::is_root() {
+                scrap::hwcodec::start_check_process(true);
+            }
+        }
         _ => {}
     }
 }
@@ -926,6 +935,12 @@ pub async fn connect_to_user_session(usid: Option<u32>) -> ResultType<()> {
     Ok(())
 }
 
+#[tokio::main(flavor = "current_thread")]
+pub async fn notify_server_to_check_hwcodec() -> ResultType<()> {
+    connect(1_000, "").await?.send(&&Data::CheckHwcodec).await?;
+    Ok(())
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

src/server.rs

@@ -449,7 +449,8 @@ pub async fn start_server(is_server: bool) {
         log::info!("XAUTHORITY={:?}", std::env::var("XAUTHORITY"));
     }
     #[cfg(feature = "hwcodec")]
-    scrap::hwcodec::hwcodec_new_check_process();
+    #[cfg(any(target_os = "windows", target_os = "linux"))]
+    scrap::hwcodec::start_check_process(false);
     #[cfg(windows)]
     hbb_common::platform::windows::start_cpu_performance_monitor();
 

src/server/video_service.rs

@@ -715,9 +715,6 @@ fn handle_hw_encoder(
         #[cfg(feature = "hwcodec")]
         match _name {
             CodecName::H264VRAM | CodecName::H265VRAM => {
-                if !scrap::codec::enable_hwcodec_option() {
-                    return Err(());
-                }
                 let is_h265 = _name == CodecName::H265VRAM;
                 let best = HwRamEncoder::best();
                 if let Some(h264) = best.h264 {

src/ui.rs

@@ -621,6 +621,10 @@ impl UI {
         );
         format!("data:image/png;base64,{s}")
     }
+
+    pub fn check_hwcodec(&self) {
+        check_hwcodec()
+    }
 }
 
 impl sciter::EventHandler for UI {
@@ -711,6 +715,7 @@ impl sciter::EventHandler for UI {
         fn generate2fa();
         fn generate_2fa_img_src(String);
         fn verify2fa(String);
+        fn check_hwcodec();
     }
 }
 

src/ui/index.tis

@@ -240,7 +240,11 @@ class Enhancements: Reactor.Component {
     event click $(menu#enhancements-menu>li) (_, me) {
         var v = me.id;
         if (v.indexOf("enable-") == 0) {
-            handler.set_option(v, handler.get_option(v) != 'N' ? 'N' : '');
+            var set_value = handler.get_option(v) != 'N' ? 'N' : '';
+            handler.set_option(v, set_value);
+            if (v == "enable-hwcodec" && set_value == '') {
+                handler.check_hwcodec();
+            }
         } else if (v.indexOf("allow-") == 0) {
             handler.set_option(v, handler.get_option(v) == 'Y' ? '' : 'Y');
         } else if (v == 'screen-recording') {

src/ui_interface.rs

@@ -829,10 +829,9 @@ pub fn get_api_server() -> String {
 
 #[inline]
 pub fn has_hwcodec() -> bool {
-    #[cfg(not(any(feature = "hwcodec", feature = "mediacodec")))]
+    // Has real hardware codec using gpu
-    return false;
+    (cfg!(feature = "hwcodec") && (cfg!(windows) || cfg!(target_os = "linux")))
-    #[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
+        || cfg!(feature = "mediacodec")
-    return true;
 }
 
 #[inline]
@@ -1315,3 +1314,14 @@ pub fn verify2fa(code: String) -> bool {
     }
     res
 }
+
+pub fn check_hwcodec() {
+    #[cfg(feature = "hwcodec")]
+    #[cfg(any(target_os = "windows", target_os = "linux"))]
+    {
+        scrap::hwcodec::start_check_process(true);
+        if crate::platform::is_installed() {
+            ipc::notify_server_to_check_hwcodec().ok();
+        }
+    }
+}