Merge pull request #2360 from fufesou/fix_wayland_ubuntu_22

embed cursor when peer is wayland

flutter/lib/desktop/pages/remote_page.dart

@@ -235,12 +235,14 @@ class _RemotePageState extends State<RemotePage>
       }))
     ];
 
-    paints.add(Obx(() => Visibility(
-        visible: _showRemoteCursor.isTrue && _remoteCursorMoved.isTrue,
-        child: CursorPaint(
-          id: widget.id,
-          zoomCursor: _zoomCursor,
-        ))));
+    if (!_ffi.canvasModel.cursorEmbeded) {
+      paints.add(Obx(() => Visibility(
+          visible: _showRemoteCursor.isTrue && _remoteCursorMoved.isTrue,
+          child: CursorPaint(
+            id: widget.id,
+            zoomCursor: _zoomCursor,
+          ))));
+    }
     paints.add(QualityMonitor(_ffi.qualityMonitorModel));
     paints.add(RemoteMenubar(
       id: widget.id,
@@ -300,20 +302,22 @@ class _ImagePaintState extends State<ImagePaint> {
 
     mouseRegion({child}) => Obx(() => MouseRegion(
         cursor: cursorOverImage.isTrue
-            ? keyboardEnabled.isTrue
-                ? (() {
-                    if (remoteCursorMoved.isTrue) {
-                      _lastRemoteCursorMoved = true;
-                      return SystemMouseCursors.none;
-                    } else {
-                      if (_lastRemoteCursorMoved) {
-                        _lastRemoteCursorMoved = false;
-                        _firstEnterImage.value = true;
-                      }
-                      return _buildCustomCursor(context, s);
-                    }
-                  }())
-                : _buildDisabledCursor(context, s)
+            ? c.cursorEmbeded
+                ? SystemMouseCursors.none
+                : keyboardEnabled.isTrue
+                    ? (() {
+                        if (remoteCursorMoved.isTrue) {
+                          _lastRemoteCursorMoved = true;
+                          return SystemMouseCursors.none;
+                        } else {
+                          if (_lastRemoteCursorMoved) {
+                            _lastRemoteCursorMoved = false;
+                            _firstEnterImage.value = true;
+                          }
+                          return _buildCustomCursor(context, s);
+                        }
+                      }())
+                    : _buildDisabledCursor(context, s)
             : MouseCursor.defer,
         onHover: (evt) {},
         child: child));

flutter/lib/desktop/pages/remote_tab_page.dart

@@ -255,8 +255,11 @@ class _ConnectionTabPageState extends State<ConnectionTabPage> {
         },
         padding: padding,
       ),
-      MenuEntryDivider<String>(),
-      () {
+    ]);
+
+    if (!ffi.canvasModel.cursorEmbeded) {
+      menu.add(MenuEntryDivider<String>());
+      menu.add(() {
         final state = ShowRemoteCursorState.find(key);
         return MenuEntrySwitch2<String>(
           switchType: SwitchType.scheckbox,
@@ -272,8 +275,8 @@ class _ConnectionTabPageState extends State<ConnectionTabPage> {
           },
           padding: padding,
         );
-      }()
-    ]);
+      }());
+    }
 
     if (perms['keyboard'] != false) {
       if (perms['clipboard'] != false) {

flutter/lib/desktop/widgets/remote_menubar.dart

@@ -1088,23 +1088,25 @@ class _RemoteMenubarState extends State<RemoteMenubar> {
     }
 
     /// Show remote cursor
-    displayMenu.add(() {
-      final state = ShowRemoteCursorState.find(widget.id);
-      return MenuEntrySwitch2<String>(
-        switchType: SwitchType.scheckbox,
-        text: translate('Show remote cursor'),
-        getter: () {
-          return state;
-        },
-        setter: (bool v) async {
-          state.value = v;
-          await bind.sessionToggleOption(
-              id: widget.id, value: 'show-remote-cursor');
-        },
-        padding: padding,
-        dismissOnClicked: true,
-      );
-    }());
+    if (!widget.ffi.canvasModel.cursorEmbeded) {
+      displayMenu.add(() {
+        final state = ShowRemoteCursorState.find(widget.id);
+        return MenuEntrySwitch2<String>(
+          switchType: SwitchType.scheckbox,
+          text: translate('Show remote cursor'),
+          getter: () {
+            return state;
+          },
+          setter: (bool v) async {
+            state.value = v;
+            await bind.sessionToggleOption(
+                id: widget.id, value: 'show-remote-cursor');
+          },
+          padding: padding,
+          dismissOnClicked: true,
+        );
+      }());
+    }
 
     /// Show remote cursor scaling with image
     if (widget.state.viewStyle.value != kRemoteViewStyleOriginal) {

flutter/lib/mobile/pages/remote_page.dart

@@ -494,38 +494,45 @@ class _RemotePageState extends State<RemotePage> {
   Widget getBodyForMobile() {
     return Container(
         color: MyTheme.canvasColor,
-        child: Stack(children: [
-          ImagePaint(),
-          CursorPaint(),
-          QualityMonitor(gFFI.qualityMonitorModel),
-          getHelpTools(),
-          SizedBox(
-            width: 0,
-            height: 0,
-            child: !_showEdit
-                ? Container()
-                : TextFormField(
-                    textInputAction: TextInputAction.newline,
-                    autocorrect: false,
-                    enableSuggestions: false,
-                    autofocus: true,
-                    focusNode: _mobileFocusNode,
-                    maxLines: null,
-                    initialValue: _value,
-                    // trick way to make backspace work always
-                    keyboardType: TextInputType.multiline,
-                    onChanged: handleSoftKeyboardInput,
-                  ),
-          ),
-        ]));
+        child: Stack(children: () {
+          final paints = [
+            ImagePaint(),
+            QualityMonitor(gFFI.qualityMonitorModel),
+            getHelpTools(),
+            SizedBox(
+              width: 0,
+              height: 0,
+              child: !_showEdit
+                  ? Container()
+                  : TextFormField(
+                      textInputAction: TextInputAction.newline,
+                      autocorrect: false,
+                      enableSuggestions: false,
+                      autofocus: true,
+                      focusNode: _mobileFocusNode,
+                      maxLines: null,
+                      initialValue: _value,
+                      // trick way to make backspace work always
+                      keyboardType: TextInputType.multiline,
+                      onChanged: handleSoftKeyboardInput,
+                    ),
+            ),
+          ];
+          if (!gFFI.canvasModel.cursorEmbeded) {
+            paints.add(CursorPaint());
+          }
+          return paints;
+        }()));
   }
 
   Widget getBodyForDesktopWithListener(bool keyboard) {
     var paints = <Widget>[ImagePaint()];
-    final cursor = bind.sessionGetToggleOptionSync(
-        id: widget.id, arg: 'show-remote-cursor');
-    if (keyboard || cursor) {
-      paints.add(CursorPaint());
+    if (!gFFI.canvasModel.cursorEmbeded) {
+      final cursor = bind.sessionGetToggleOptionSync(
+          id: widget.id, arg: 'show-remote-cursor');
+      if (keyboard || cursor) {
+        paints.add(CursorPaint());
+      }
     }
     return Container(
         color: MyTheme.canvasColor, child: Stack(children: paints));
@@ -1046,9 +1053,12 @@ void showOptions(
     }
 
     final toggles = [
-      getToggle(id, setState, 'show-remote-cursor', 'Show remote cursor'),
       getToggle(id, setState, 'show-quality-monitor', 'Show quality monitor'),
     ];
+    if (!gFFI.canvasModel.cursorEmbeded) {
+      toggles.insert(0,
+          getToggle(id, setState, 'show-remote-cursor', 'Show remote cursor'));
+    }
 
     return CustomAlertDialog(
       content: Column(

flutter/lib/models/model.dart

@@ -221,6 +221,7 @@ class FfiModel with ChangeNotifier {
     _display.y = double.parse(evt['y']);
     _display.width = int.parse(evt['width']);
     _display.height = int.parse(evt['height']);
+    _display.cursorEmbeded = int.parse(evt['cursor_embeded']) == 1;
     if (old != _pi.currentDisplay) {
       parent.target?.cursorModel.updateDisplayOrigin(_display.x, _display.y);
     }
@@ -330,6 +331,7 @@ class FfiModel with ChangeNotifier {
         d.y = d0['y'].toDouble();
         d.width = d0['width'];
         d.height = d0['height'];
+        d.cursorEmbeded = d0['cursor_embeded'] == 1;
         _pi.displays.add(d);
       }
       if (_pi.currentDisplay < _pi.displays.length) {
@@ -582,6 +584,9 @@ class CanvasModel with ChangeNotifier {
     notifyListeners();
   }
 
+  bool get cursorEmbeded =>
+      parent.target?.ffiModel.display.cursorEmbeded ?? false;
+
   int getDisplayWidth() {
     final defaultWidth = (isDesktop || isWebDesktop)
         ? kDesktopDefaultDisplayWidth
@@ -1311,6 +1316,7 @@ class Display {
   double y = 0;
   int width = 0;
   int height = 0;
+  bool cursorEmbeded = false;
 
   Display() {
     width = (isDesktop || isWebDesktop)

libs/hbb_common/protos/message.proto

@@ -40,6 +40,7 @@ message DisplayInfo {
   int32 height = 4;
   string name = 5;
   bool online = 6;
+  bool cursor_embeded = 7;
 }
 
 message PortForward {
@@ -419,6 +420,7 @@ message SwitchDisplay {
   sint32 y = 3;
   int32 width = 4;
   int32 height = 5;
+  bool cursor_embeded = 6;
 }
 
 message PermissionInfo {

libs/scrap/src/common/mod.rs

@@ -69,3 +69,19 @@ pub trait TraitCapturer {
 pub fn is_x11() -> bool {
     "x11" == hbb_common::platform::linux::get_display_server()
 }
+
+#[cfg(x11)]
+#[inline]
+pub fn is_cursor_embeded() -> bool {
+    if is_x11() {
+        x11::IS_CURSOR_EMBEDED
+    } else {
+        wayland::IS_CURSOR_EMBEDED
+    }
+}
+
+#[cfg(not(x11))]
+#[inline]
+pub fn is_cursor_embeded() -> bool {
+    false
+}

libs/scrap/src/common/wayland.rs

@@ -4,6 +4,8 @@ use std::{io, sync::RwLock, time::Duration};
 
 pub struct Capturer(Display, Box<dyn Recorder>, bool, Vec<u8>);
 
+pub const IS_CURSOR_EMBEDED: bool = true;
+
 lazy_static::lazy_static! {
     static ref MAP_ERR: RwLock<Option<fn(err: String)-> io::Error>> = Default::default();
 }
@@ -66,7 +68,7 @@ impl Display {
     }
 
     pub fn all() -> io::Result<Vec<Display>> {
-        Ok(pipewire::get_capturables(false)
+        Ok(pipewire::get_capturables(true)
             .map_err(map_err)?
             .drain(..)
             .map(|x| Display(x))

libs/scrap/src/common/x11.rs

@@ -3,6 +3,8 @@ use std::{io, ops, time::Duration};
 
 pub struct Capturer(x11::Capturer);
 
+pub const IS_CURSOR_EMBEDED: bool = false;
+
 impl Capturer {
     pub fn new(display: Display, yuv: bool) -> io::Result<Capturer> {
         x11::Capturer::new(display.0, yuv).map(Capturer)

src/client/io_loop.rs

@@ -976,7 +976,7 @@ impl<T: InvokeUiSession> Remote<T> {
                         self.handler.ui_handler.switch_display(&s);
                         self.video_sender.send(MediaData::Reset).ok();
                         if s.width > 0 && s.height > 0 {
-                            self.handler.set_display(s.x, s.y, s.width, s.height);
+                            self.handler.set_display(s.x, s.y, s.width, s.height, s.cursor_embeded);
                         }
                     }
                     Some(misc::Union::CloseReason(c)) => {

src/flutter.rs

@@ -155,7 +155,7 @@ impl InvokeUiSession for FlutterHandler {
     }
 
     /// unused in flutter, use switch_display or set_peer_info
-    fn set_display(&self, _x: i32, _y: i32, _w: i32, _h: i32) {}
+    fn set_display(&self, _x: i32, _y: i32, _w: i32, _h: i32, _cursor_embeded: bool) {}
 
     fn update_privacy_mode(&self) {
         self.push_event("update_privacy_mode", [].into());
@@ -295,6 +295,7 @@ impl InvokeUiSession for FlutterHandler {
             h.insert("y", d.y);
             h.insert("width", d.width);
             h.insert("height", d.height);
+            h.insert("cursor_embeded", if d.cursor_embeded { 1 } else { 0 });
             displays.push(h);
         }
         let displays = serde_json::ser::to_string(&displays).unwrap_or("".to_owned());
@@ -343,6 +344,7 @@ impl InvokeUiSession for FlutterHandler {
                 ("y", &display.y.to_string()),
                 ("width", &display.width.to_string()),
                 ("height", &display.height.to_string()),
+                ("cursor_embeded", &{if display.cursor_embeded {1} else {0}}.to_string()),
             ],
         );
     }

src/lang.rs

@@ -110,7 +110,7 @@ pub fn translate_locale(name: String, locale: &str) -> String {
         "fa" => fa::T.deref(),
         "ca" => ca::T.deref(),
         "gr" => gr::T.deref(),
-        "gr" => sv::T.deref(),
+        "sv" => sv::T.deref(),
         _ => en::T.deref(),
     };
     if let Some(v) = m.get(&name as &str) {

src/server.rs

@@ -85,8 +85,10 @@ pub fn new() -> ServerPtr {
     #[cfg(not(any(target_os = "android", target_os = "ios")))]
     {
         server.add_service(Box::new(clipboard_service::new()));
-        server.add_service(Box::new(input_service::new_cursor()));
-        server.add_service(Box::new(input_service::new_pos()));
+        if !video_service::capture_cursor_embeded() {
+            server.add_service(Box::new(input_service::new_cursor()));
+            server.add_service(Box::new(input_service::new_pos()));
+        }
     }
     Arc::new(RwLock::new(server))
 }

src/server/video_service.rs

@@ -74,6 +74,10 @@ fn is_capturer_mag_supported() -> bool {
     false
 }
 
+pub fn capture_cursor_embeded() -> bool {
+    scrap::is_cursor_embeded()
+}
+
 pub fn notify_video_frame_feched(conn_id: i32, frame_tm: Option<Instant>) {
     FRAME_FETCHED_NOTIFIER.0.send((conn_id, frame_tm)).unwrap()
 }
@@ -455,6 +459,7 @@ fn run(sp: GenericService) -> ResultType<()> {
             y: c.origin.1 as _,
             width: c.width as _,
             height: c.height as _,
+            cursor_embeded: capture_cursor_embeded(),
             ..Default::default()
         });
         let mut msg_out = Message::new();
@@ -783,6 +788,7 @@ pub(super) fn get_displays_2(all: &Vec<Display>) -> (usize, Vec<DisplayInfo>) {
             height: d.height() as _,
             name: d.name(),
             online: d.is_online(),
+            cursor_embeded: false,
             ..Default::default()
         });
     }

src/server/wayland.rs

@@ -127,7 +127,10 @@ pub(super) async fn check_init() -> ResultType<()> {
             if *lock == 0 {
                 let all = Display::all()?;
                 let num = all.len();
-                let (primary, displays) = super::video_service::get_displays_2(&all);
+                let (primary, mut displays) = super::video_service::get_displays_2(&all);
+                for display in displays.iter_mut() {
+                    display.cursor_embeded = true;
+                }
 
                 let mut rects: Vec<((i32, i32), usize, usize)> = Vec::new();
                 for d in &all {

src/ui/header.tis

@@ -164,6 +164,13 @@ class Header: Reactor.Component {
         var codecs = handler.supported_hwcodec();
         var show_codec = handler.has_hwcodec() && (codecs[0] || codecs[1]);
 
+        var cursor_embeded = false;
+        if ((pi.displays || []).length > 0) {
+            if (pi.displays.length > pi.current_display) {
+                cursor_embeded = pi.displays[pi.current_display].cursor_embeded;
+            }
+        }
+
         return <popup>
             <menu.context #display-options>
                 <li #adjust-window style="display:none">{translate('Adjust Window')}</li> 
@@ -184,7 +191,7 @@ class Header: Reactor.Component {
                 {codecs[1] ? <li #h265 type="codec-preference"><span>{svg_checkmark}</span>H265</li> : ""}
                 </div> : ""}
                 <div .separator />
-                <li #show-remote-cursor .toggle-option><span>{svg_checkmark}</span>{translate('Show remote cursor')}</li> 
+                {!cursor_embeded && <li #show-remote-cursor .toggle-option><span>{svg_checkmark}</span>{translate('Show remote cursor')}</li>}
                 <li #show-quality-monitor .toggle-option><span>{svg_checkmark}</span>{translate('Show quality monitor')}</li> 
                 {audio_enabled ? <li #disable-audio .toggle-option><span>{svg_checkmark}</span>{translate('Mute')}</li> : ""}
                 {is_win && pi.platform == 'Windows' && file_enabled ? <li #enable-file-transfer .toggle-option><span>{svg_checkmark}</span>{translate('Allow file copy and paste')}</li> : ""}

src/ui/remote.rs

@@ -79,8 +79,8 @@ impl InvokeUiSession for SciterHandler {
         }
     }
 
-    fn set_display(&self, x: i32, y: i32, w: i32, h: i32) {
-        self.call("setDisplay", &make_args!(x, y, w, h));
+    fn set_display(&self, x: i32, y: i32, w: i32, h: i32, cursor_embeded: bool) {
+        self.call("setDisplay", &make_args!(x, y, w, h, cursor_embeded));
         // https://sciter.com/forums/topic/color_spaceiyuv-crash
         // Nothing spectacular in decoder – done on CPU side.
         // So if you can do BGRA translation on your side – the better.
@@ -223,6 +223,7 @@ impl InvokeUiSession for SciterHandler {
             display.set_item("y", d.y);
             display.set_item("width", d.width);
             display.set_item("height", d.height);
+            display.set_item("cursor_embeded", d.cursor_embeded);
             displays.push(display);
         }
         pi_sciter.set_item("displays", displays);

src/ui/remote.tis

@@ -6,6 +6,7 @@ var display_width = 0;
 var display_height = 0;
 var display_origin_x = 0;
 var display_origin_y = 0;
+var display_cursor_embeded = false;
 var display_scale = 1;
 var keyboard_enabled = true; // server side
 var clipboard_enabled = true; // server side
@@ -15,11 +16,12 @@ var restart_enabled = true; // server side
 var recording_enabled = true; // server side
 var scroll_body = $(body);
 
-handler.setDisplay = function(x, y, w, h) {
+handler.setDisplay = function(x, y, w, h, cursor_embeded) {
     display_width = w;
     display_height = h;
     display_origin_x = x;
     display_origin_y = y;
+    display_cursor_embeded = cursor_embeded;
     adaptDisplay();
     if (recording) handler.record_screen(true, w, h);
 }
@@ -195,6 +197,9 @@ function handler.onMouse(evt)
         dragging = false;
         break;
       case Event.MOUSE_MOVE:
+        if (display_cursor_embeded) {
+            break;
+        }
         if (cursor_img.style#display != "none" && keyboard_enabled) {
             cursor_img.style#display = "none";
         }
@@ -360,6 +365,10 @@ function updateCursor(system=false) {
 }
 
 function refreshCursor() {
+    if (display_cursor_embeded) {
+        cursor_img.style#display = "none";
+        return;
+    }
     if (cur_id != -1) {
         handler.setCursorId(cur_id);
     }

src/ui_session_interface.rs

@@ -1098,7 +1098,7 @@ pub trait InvokeUiSession: Send + Sync + Clone + 'static + Sized + Default {
     fn set_cursor_data(&self, cd: CursorData);
     fn set_cursor_id(&self, id: String);
     fn set_cursor_position(&self, cp: CursorPosition);
-    fn set_display(&self, x: i32, y: i32, w: i32, h: i32);
+    fn set_display(&self, x: i32, y: i32, w: i32, h: i32, cursor_embeded: bool);
     fn switch_display(&self, display: &SwitchDisplay);
     fn set_peer_info(&self, peer_info: &PeerInfo); // flutter
     fn update_privacy_mode(&self);
@@ -1211,7 +1211,7 @@ impl<T: InvokeUiSession> Interface for Session<T> {
                 input_os_password(p, true, self.clone());
             }
             let current = &pi.displays[pi.current_display as usize];
-            self.set_display(current.x, current.y, current.width, current.height);
+            self.set_display(current.x, current.y, current.width, current.height, current.cursor_embeded);
         }
         self.update_privacy_mode();
         // Save recent peers, then push event to flutter. So flutter can refresh peer page.