Refact/multi window soft rendering (#8343)

* refact: multi_window_soft_rendering Signed-off-by: fufesou <linlong1266@gmail.com> * fix: window pos, potential wait for image Signed-off-by: fufesou <linlong1266@gmail.com> * comments Signed-off-by: fufesou <linlong1266@gmail.com> * remove debug print Signed-off-by: fufesou <linlong1266@gmail.com> * explicitly set rgba_data.size_got to false after init Signed-off-by: fufesou <linlong1266@gmail.com> * refact: multi window, merge images, render with texture Signed-off-by: fufesou <linlong1266@gmail.com> * revert, flutter.rs, rgba valid Signed-off-by: fufesou <linlong1266@gmail.com> * Add displays index before sending capture msg Signed-off-by: fufesou <linlong1266@gmail.com> * refact: multi window, soft rendering Signed-off-by: fufesou <linlong1266@gmail.com> * fix: build Signed-off-by: fufesou <linlong1266@gmail.com> --------- Signed-off-by: fufesou <linlong1266@gmail.com>
2024-11-27 14:59:02 +08:00 · 2024-06-13 18:03:41 +08:00 · 2024-06-13 18:03:41 +08:00 · bc875a35b0
commit bc875a35b0
parent 8e12a34634
10 changed files with 160 additions and 82 deletions
--- a/flutter/lib/common.dart
+++ b/flutter/lib/common.dart
@ -2924,10 +2924,10 @@ openMonitorInNewTabOrWindow(int i, String peerId, PeerInfo pi,
      kMainWindowId, kWindowEventOpenMonitorSession, jsonEncode(args));
 }

-setNewConnectWindowFrame(int windowId, String peerId, Rect? screenRect) async {
+setNewConnectWindowFrame(int windowId, String peerId, int? display, Rect? screenRect) async {
  if (screenRect == null) {
    await restoreWindowPosition(WindowType.RemoteDesktop,
-        windowId: windowId, peerId: peerId);
+        windowId: windowId, display: display, peerId: peerId);
  } else {
    await tryMoveToScreenAndSetFullscreen(screenRect);
  }
--- a/flutter/lib/common/widgets/setting_widgets.dart
+++ b/flutter/lib/common/widgets/setting_widgets.dart
@ -234,12 +234,12 @@ List<(String, String)> otherDefaultSettings() {
    ('True color (4:4:4)', kOptionI444),
    ('Reverse mouse wheel', kKeyReverseMouseWheel),
    ('swap-left-right-mouse', kOptionSwapLeftRightMouse),
-    if (isDesktop && bind.mainGetUseTextureRender())
+    if (isDesktop)
      (
        'Show displays as individual windows',
        kKeyShowDisplaysAsIndividualWindows
      ),
-    if (isDesktop && bind.mainGetUseTextureRender())
+    if (isDesktop)
      (
        'Use all my displays for the remote session',
        kKeyUseAllMyDisplaysForTheRemoteSession
--- a/flutter/lib/common/widgets/toolbar.dart
+++ b/flutter/lib/common/widgets/toolbar.dart
@ -586,7 +586,6 @@ Future<List<TToggleMenu>> toolbarDisplayToggle(
  if (pi.isSupportMultiDisplay &&
      PrivacyModeState.find(id).isEmpty &&
      pi.displaysCount.value > 1 &&
-      bind.mainGetUseTextureRender() &&
      bind.mainGetUserDefaultOption(key: kKeyShowMonitorsToolbar) == 'Y') {
    final value =
        bind.sessionGetDisplaysAsIndividualWindows(sessionId: ffi.sessionId) ==
@ -602,9 +601,7 @@ Future<List<TToggleMenu>> toolbarDisplayToggle(
  }

  final isMultiScreens = !isWeb && (await getScreenRectList()).length > 1;
-  if (bind.mainGetUseTextureRender() &&
-      pi.isSupportMultiDisplay &&
-      isMultiScreens) {
+  if (pi.isSupportMultiDisplay && isMultiScreens) {
    final value = bind.sessionGetUseAllMyDisplaysForTheRemoteSession(
            sessionId: ffi.sessionId) ==
        'Y';
--- a/flutter/lib/desktop/pages/remote_page.dart
+++ b/flutter/lib/desktop/pages/remote_page.dart
@ -617,10 +617,11 @@ class _ImagePaintState extends State<ImagePaint> {
      final paintWidth = c.getDisplayWidth() * s;
      final paintHeight = c.getDisplayHeight() * s;
      final paintSize = Size(paintWidth, paintHeight);
-      final paintWidget = m.useTextureRender
-          ? _BuildPaintTextureRender(
-              c, s, Offset.zero, paintSize, isViewOriginal())
-          : _buildScrollbarNonTextureRender(m, paintSize, s);
+      final paintWidget =
+          m.useTextureRender || widget.ffi.ffiModel.pi.forceTextureRender
+              ? _BuildPaintTextureRender(
+                  c, s, Offset.zero, paintSize, isViewOriginal())
+              : _buildScrollbarNonTextureRender(m, paintSize, s);
      return NotificationListener<ScrollNotification>(
          onNotification: (notification) {
            c.updateScrollPercent();
@ -638,17 +639,18 @@ class _ImagePaintState extends State<ImagePaint> {
          ));
    } else {
      if (c.size.width > 0 && c.size.height > 0) {
-        final paintWidget = m.useTextureRender
-            ? _BuildPaintTextureRender(
-                c,
-                s,
-                Offset(
-                  isLinux ? c.x.toInt().toDouble() : c.x,
-                  isLinux ? c.y.toInt().toDouble() : c.y,
-                ),
-                c.size,
-                isViewOriginal())
-            : _buildScrollAuthNonTextureRender(m, c, s);
+        final paintWidget =
+            m.useTextureRender || widget.ffi.ffiModel.pi.forceTextureRender
+                ? _BuildPaintTextureRender(
+                    c,
+                    s,
+                    Offset(
+                      isLinux ? c.x.toInt().toDouble() : c.x,
+                      isLinux ? c.y.toInt().toDouble() : c.y,
+                    ),
+                    c.size,
+                    isViewOriginal())
+                : _buildScrollAutoNonTextureRender(m, c, s);
        return mouseRegion(child: _buildListener(paintWidget));
      } else {
        return Container();
@ -664,7 +666,7 @@ class _ImagePaintState extends State<ImagePaint> {
    );
  }

-  Widget _buildScrollAuthNonTextureRender(
+  Widget _buildScrollAutoNonTextureRender(
      ImageModel m, CanvasModel c, double s) {
    return CustomPaint(
      size: Size(c.size.width, c.size.height),
--- a/flutter/lib/desktop/pages/remote_tab_page.dart
+++ b/flutter/lib/desktop/pages/remote_tab_page.dart
@ -420,7 +420,7 @@ class _ConnectionTabPageState extends State<ConnectionTabPage> {
          await WindowController.fromWindowId(windowId()).setFullscreen(false);
          stateGlobal.setFullscreen(false, procWnd: false);
        }
-        await setNewConnectWindowFrame(windowId(), id!, screenRect);
+        await setNewConnectWindowFrame(windowId(), id!, display, screenRect);
        Future.delayed(Duration(milliseconds: isWindows ? 100 : 0), () async {
          await windowOnTop(windowId());
        });
--- a/flutter/lib/desktop/widgets/remote_toolbar.dart
+++ b/flutter/lib/desktop/widgets/remote_toolbar.dart
@ -643,15 +643,12 @@ class _MonitorMenu extends StatelessWidget {
  }

  Widget buildMonitorSubmenuWidget(BuildContext context) {
-    final m = Provider.of<ImageModel>(context);
    return Column(
      mainAxisSize: MainAxisSize.min,
      children: [
        Row(children: buildMonitorList(context, false)),
-        supportIndividualWindows && m.useTextureRender ? Divider() : Offstage(),
-        supportIndividualWindows && m.useTextureRender
-            ? chooseDisplayBehavior()
-            : Offstage(),
+        supportIndividualWindows ? Divider() : Offstage(),
+        supportIndividualWindows ? chooseDisplayBehavior() : Offstage(),
      ],
    );
  }
@ -737,10 +734,7 @@ class _MonitorMenu extends StatelessWidget {
    for (int i = 0; i < pi.displays.length; i++) {
      monitorList.add(buildMonitorButton(i));
    }
-    final m = Provider.of<ImageModel>(context);
-    if (supportIndividualWindows &&
-        m.useTextureRender &&
-        pi.displays.length > 1) {
+    if (supportIndividualWindows && pi.displays.length > 1) {
      monitorList.add(buildMonitorButton(kAllDisplayValue));
    }
    return monitorList;
@ -824,7 +818,6 @@ class _MonitorMenu extends StatelessWidget {
    RxInt display = CurrentDisplayState.find(id);
    if (display.value != i) {
      final isChooseDisplayToOpenInNewWindow = pi.isSupportMultiDisplay &&
-          bind.mainGetUseTextureRender() &&
          bind.sessionGetDisplaysAsIndividualWindows(
                  sessionId: ffi.sessionId) ==
              'Y';
--- a/flutter/lib/models/model.dart
+++ b/flutter/lib/models/model.dart
@ -1187,10 +1187,11 @@ class ImageModel with ChangeNotifier {

  onRgba(int display, Uint8List rgba) {
    final pid = parent.target?.id;
+    final rect = parent.target?.ffiModel.pi.getDisplayRect(display);
    img.decodeImageFromPixels(
        rgba,
-        parent.target?.ffiModel.rect?.width.toInt() ?? 0,
-        parent.target?.ffiModel.rect?.height.toInt() ?? 0,
+        rect?.width.toInt() ?? 0,
+        rect?.height.toInt() ?? 0,
        isWeb ? ui.PixelFormat.rgba8888 : ui.PixelFormat.bgra8888,
        onPixelsCopied: () {
      // Unlock the rgba memory from rust codes.
@ -2397,9 +2398,10 @@ class FFI {
      cursorModel.peerId = id;
    }

+    final isNewPeer = tabWindowId == null;
    // If tabWindowId != null, this session is a "tab -> window" one.
    // Else this session is a new one.
-    if (tabWindowId == null) {
+    if (isNewPeer) {
      // ignore: unused_local_variable
      final addRes = bind.sessionAddSync(
        sessionId: sessionId,
@ -2424,14 +2426,25 @@ class FFI {
            'Unreachable, failed to add existed session to $id, $addRes');
        return;
      }
-      bind.sessionTryAddDisplay(
-          sessionId: sessionId, displays: Int32List.fromList(displays));
      ffiModel.pi.currentDisplay = display;
    }
    if (isDesktop && connType == ConnType.defaultConn) {
      textureModel.updateCurrentDisplay(display ?? 0);
    }
-    final stream = bind.sessionStart(sessionId: sessionId, id: id);
+
+    // CAUTION: `sessionStart()` and `sessionStartWithDisplays()` are an async functions.
+    // Though the stream is returned immediately, the stream may not be ready.
+    // Any operations that depend on the stream should be carefully handled.
+    late final Stream<EventToUI> stream;
+    if (isNewPeer || display == null || displays == null) {
+      stream = bind.sessionStart(sessionId: sessionId, id: id);
+    } else {
+      // We have to put displays in `sessionStart()` to make sure the stream is ready
+      // and then the displays' capturing requests can be sent.
+      stream = bind.sessionStartWithDisplays(
+          sessionId: sessionId, id: id, displays: Int32List.fromList(displays));
+    }
+
    if (isWeb) {
      platformFFI.setRgbaCallback((int display, Uint8List data) {
        onEvent2UIRgba();
@ -2442,14 +2455,6 @@ class FFI {

    final cb = ffiModel.startEventListener(sessionId, id);

-    // Force refresh displays.
-    // The controlled side may not refresh the image when the (peer,display) is already subscribed.
-    if (displays != null) {
-      for (final display in displays) {
-        bind.sessionRefresh(sessionId: sessionId, display: display);
-      }
-    }
-
    imageModel.updateUserTextureRender();
    final hasGpuTextureRender = bind.mainHasGpuTextureRender();
    final SimpleWrapper<bool> isToNewWindowNotified = SimpleWrapper(false);
@ -2500,8 +2505,8 @@ class FFI {
          }
        } else if (message is EventToUI_Rgba) {
          final display = message.field0;
-          if (imageModel.useTextureRender) {
-            debugPrint("EventToUI_Rgba display:$display");
+          if (imageModel.useTextureRender || ffiModel.pi.forceTextureRender) {
+            //debugPrint("EventToUI_Rgba display:$display");
            textureModel.setTextureType(display: display, gpuTexture: false);
            onEvent2UIRgba();
          } else {
@ -2692,6 +2697,7 @@ class PeerInfo with ChangeNotifier {

  bool get isSupportMultiDisplay =>
      (isDesktop || isWebDesktop) && isSupportMultiUiSession;
+  bool get forceTextureRender => currentDisplay == kAllDisplayValue;

  bool get cursorEmbedded => tryGetDisplay()?.cursorEmbedded ?? false;

@ -2700,30 +2706,32 @@ class PeerInfo with ChangeNotifier {
  bool get isAmyuniIdd =>
      platformAdditions[kPlatformAdditionsIddImpl] == 'amyuni_idd';

-  Display? tryGetDisplay() {
+  Display? tryGetDisplay({int? display}) {
    if (displays.isEmpty) {
      return null;
    }
-    if (currentDisplay == kAllDisplayValue) {
+    display ??= currentDisplay;
+    if (display == kAllDisplayValue) {
      return displays[0];
    } else {
-      if (currentDisplay > 0 && currentDisplay < displays.length) {
-        return displays[currentDisplay];
+      if (display > 0 && display < displays.length) {
+        return displays[display];
      } else {
        return displays[0];
      }
    }
  }

-  Display? tryGetDisplayIfNotAllDisplay() {
+  Display? tryGetDisplayIfNotAllDisplay({int? display}) {
    if (displays.isEmpty) {
      return null;
    }
-    if (currentDisplay == kAllDisplayValue) {
+    display ??= currentDisplay;
+    if (display == kAllDisplayValue) {
      return null;
    }
-    if (currentDisplay >= 0 && currentDisplay < displays.length) {
-      return displays[currentDisplay];
+    if (display >= 0 && display < displays.length) {
+      return displays[display];
    } else {
      return null;
    }
@ -2747,6 +2755,12 @@ class PeerInfo with ChangeNotifier {
    }
    return 1.0;
  }
+
+  Rect? getDisplayRect(int display) {
+    final d = tryGetDisplayIfNotAllDisplay(display: display);
+    if (d == null) return null;
+    return Rect.fromLTWH(d.x, d.y, d.width.toDouble(), d.height.toDouble());
+  }
 }

 const canvasKey = 'canvas';
--- a/flutter/lib/web/bridge.dart
+++ b/flutter/lib/web/bridge.dart
@ -63,11 +63,6 @@ class RustdeskImpl {
    return '';
  }

-  void sessionTryAddDisplay(
-      {required UuidValue sessionId,
-      required Int32List displays,
-      dynamic hint}) {}
-
  String sessionAddSync(
      {required UuidValue sessionId,
      required String id,
@ -94,6 +89,14 @@ class RustdeskImpl {
    return Stream.empty();
  }

+  Stream<EventToUI> sessionStartWithDisplays(
+      {required UuidValue sessionId,
+      required String id,
+      required Int32List displays,
+      dynamic hint}) {
+    throw UnimplementedError();
+  }
+
  Future<bool?> sessionGetRemember(
      {required UuidValue sessionId, dynamic hint}) {
    return Future(
--- a/src/flutter.rs
+++ b/src/flutter.rs
@ -408,6 +408,25 @@ impl VideoRenderer {
        }
    }

+    fn add_displays(&self, displays: &[i32]) {
+        let mut sessions_lock = self.map_display_sessions.write().unwrap();
+        for display in displays {
+            let d = *display as usize;
+            if !sessions_lock.contains_key(&d) {
+                sessions_lock.insert(
+                    d,
+                    DisplaySessionInfo {
+                        texture_rgba_ptr: 0,
+                        size: (0, 0),
+                        #[cfg(feature = "vram")]
+                        gpu_output_ptr: usize::default(),
+                        notify_render_type: None,
+                    },
+                );
+            }
+        }
+    }
+
    #[cfg(not(any(target_os = "android", target_os = "ios")))]
    pub fn on_rgba(&self, display: usize, rgba: &scrap::ImageRgb) -> bool {
        let mut write_lock = self.map_display_sessions.write().unwrap();
@ -768,9 +787,9 @@ impl InvokeUiSession for FlutterHandler {
    #[inline]
    #[cfg(not(any(target_os = "android", target_os = "ios")))]
    fn on_rgba(&self, display: usize, rgba: &mut scrap::ImageRgb) {
-        if self.use_texture_render.load(Ordering::Relaxed) {
-            self.on_rgba_flutter_texture_render(display, rgba);
-        } else {
+        let use_texture_render = self.use_texture_render.load(Ordering::Relaxed);
+        self.on_rgba_flutter_texture_render(use_texture_render, display, rgba);
+        if !use_texture_render {
            self.on_rgba_soft_render(display, rgba);
        }
    }
@ -1039,22 +1058,43 @@ impl FlutterHandler {
        }
        drop(rgba_write_lock);

-        // Non-texture-render UI does not support multiple displays in the one UI session.
-        // It's Ok to notify each session for now.
        for h in self.session_handlers.read().unwrap().values() {
-            if let Some(stream) = &h.event_stream {
-                stream.add(EventToUI::Rgba(display));
+            // `map_display_sessions` stores the display indices that are used by the video renderer.
+            let map_display_sessions = h.renderer.map_display_sessions.read().unwrap();
+            // The soft renderer does not support multi ui session for now.
+            if map_display_sessions.len() > 1 {
+                continue;
+            }
+            if h.renderer
+                .map_display_sessions
+                .read()
+                .unwrap()
+                .contains_key(&display)
+            {
+                if map_display_sessions.contains_key(&display) {
+                    if let Some(stream) = &h.event_stream {
+                        stream.add(EventToUI::Rgba(display));
+                    }
+                }
            }
        }
    }

    #[inline]
    #[cfg(not(any(target_os = "android", target_os = "ios")))]
-    fn on_rgba_flutter_texture_render(&self, display: usize, rgba: &mut scrap::ImageRgb) {
+    fn on_rgba_flutter_texture_render(
+        &self,
+        use_texture_render: bool,
+        display: usize,
+        rgba: &mut scrap::ImageRgb,
+    ) {
        for (_, session) in self.session_handlers.read().unwrap().iter() {
-            if session.renderer.on_rgba(display, rgba) {
-                if let Some(stream) = &session.event_stream {
-                    stream.add(EventToUI::Rgba(display));
+            if use_texture_render || session.renderer.map_display_sessions.read().unwrap().len() > 1
+            {
+                if session.renderer.on_rgba(display, rgba) {
+                    if let Some(stream) = &session.event_stream {
+                        stream.add(EventToUI::Rgba(display));
+                    }
                }
            }
        }
@ -1522,6 +1562,21 @@ pub fn session_register_gpu_texture(_session_id: SessionID, _display: usize, _ou
    }
 }

+pub fn session_add_displays(session_id: &SessionID, displays: &[i32]) {
+    for s in sessions::get_sessions() {
+        if let Some(h) = s
+            .ui_handler
+            .session_handlers
+            .read()
+            .unwrap()
+            .get(session_id)
+        {
+            h.renderer.add_displays(displays);
+            break;
+        }
+    }
+}
+
 #[inline]
 #[cfg(not(feature = "vram"))]
 pub fn get_adapter_luid() -> Option<i64> {
--- a/src/flutter_ffi.rs
+++ b/src/flutter_ffi.rs
@ -110,13 +110,6 @@ pub fn session_add_existed_sync(id: String, session_id: SessionID) -> SyncReturn
    }
 }

-pub fn session_try_add_display(session_id: SessionID, displays: Vec<i32>) -> SyncReturn<()> {
-    if let Some(session) = sessions::get_session_by_session_id(&session_id) {
-        session.capture_displays(displays, vec![], vec![]);
-    }
-    SyncReturn(())
-}
-
 pub fn session_add_sync(
    session_id: SessionID,
    id: String,
@ -153,6 +146,27 @@ pub fn session_start(
    session_start_(&session_id, &id, events2ui)
 }

+pub fn session_start_with_displays(
+    events2ui: StreamSink<EventToUI>,
+    session_id: SessionID,
+    id: String,
+    displays: Vec<i32>,
+) -> ResultType<()> {
+    session_start_(&session_id, &id, events2ui)?;
+
+    // `session_add_displays` is used for software rendering, multi-ui sessions.
+    // We need to add displays to the session first, then capture the displays.
+    // Otherwise, the session may not send video frames to the flutter side.
+    super::flutter::session_add_displays(&session_id, &displays);
+    if let Some(session) = sessions::get_session_by_session_id(&session_id) {
+        session.capture_displays(displays.clone(), vec![], vec![]);
+        for display in displays {
+            session.refresh_video(display as _);
+        }
+    }
+    Ok(())
+}
+
 pub fn session_get_remember(session_id: SessionID) -> Option<bool> {
    if let Some(session) = sessions::get_session_by_session_id(&session_id) {
        Some(session.get_remember())