diff --git a/flutter/lib/common/widgets/remote_input.dart b/flutter/lib/common/widgets/remote_input.dart
index bea1490fe..c6fdaf75b 100644
--- a/flutter/lib/common/widgets/remote_input.dart
+++ b/flutter/lib/common/widgets/remote_input.dart
@@ -86,6 +86,12 @@ class _RawTouchGestureDetectorRegionState
 
   PointerDeviceKind? lastDeviceKind;
 
+  // For touch mode, onDoubleTap
+  // `onDoubleTap()` does not provide the position of the tap event.
+  Offset _lastPosOfDoubleTapDown = Offset.zero;
+  bool _touchModePanStarted = false;
+  Offset _doubleFinerTapPosition = Offset.zero;
+
   FFI get ffi => widget.ffi;
   FfiModel get ffiModel => widget.ffiModel;
   InputModel get inputModel => widget.inputModel;
@@ -106,6 +112,7 @@ class _RawTouchGestureDetectorRegionState
       return;
     }
     if (handleTouch) {
+      _lastPosOfDoubleTapDown = d.localPosition;
       // Desktop or mobile "Touch mode"
       if (ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy)) {
         inputModel.tapDown(MouseButtons.left);
@@ -140,6 +147,7 @@ class _RawTouchGestureDetectorRegionState
       return;
     }
     if (handleTouch) {
+      _lastPosOfDoubleTapDown = d.localPosition;
       ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy);
     }
   }
@@ -151,6 +159,10 @@ class _RawTouchGestureDetectorRegionState
     if (ffiModel.touchMode && ffi.cursorModel.lastIsBlocked) {
       return;
     }
+    if (handleTouch &&
+        !ffi.cursorModel.isInRemoteRect(_lastPosOfDoubleTapDown)) {
+      return;
+    }
     inputModel.tap(MouseButtons.left);
     inputModel.tap(MouseButtons.left);
   }
@@ -161,8 +173,11 @@ class _RawTouchGestureDetectorRegionState
       return;
     }
     if (handleTouch) {
-      ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy);
+      _lastPosOfDoubleTapDown = d.localPosition;
       _cacheLongPressPosition = d.localPosition;
+      if (!ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy)) {
+        return;
+      }
       _cacheLongPressPositionTs = DateTime.now().millisecondsSinceEpoch;
     }
   }
@@ -182,8 +197,10 @@ class _RawTouchGestureDetectorRegionState
       return;
     }
     if (handleTouch) {
-      ffi.cursorModel
-          .move(_cacheLongPressPosition.dx, _cacheLongPressPosition.dy);
+      if (!ffi.cursorModel
+          .move(_cacheLongPressPosition.dx, _cacheLongPressPosition.dy)) {
+        return;
+      }
     }
     if (!ffi.ffiModel.isPeerMobile) {
       inputModel.tap(MouseButtons.right);
@@ -195,6 +212,7 @@ class _RawTouchGestureDetectorRegionState
     if (lastDeviceKind != PointerDeviceKind.touch) {
       return;
     }
+    _doubleFinerTapPosition = d.localPosition;
     // ignore for desktop and mobile
   }
 
@@ -203,7 +221,13 @@ class _RawTouchGestureDetectorRegionState
     if (lastDeviceKind != PointerDeviceKind.touch) {
       return;
     }
-    if ((isDesktop || isWebDesktop) || !ffiModel.touchMode) {
+
+    // mobile mouse mode or desktop touch screen
+    final isMobileMouseMode = isMobile && !ffiModel.touchMode;
+    // We can't use `d.localPosition` here because it's always (0, 0) on desktop.
+    final isDesktopInRemoteRect = (isDesktop || isWebDesktop) &&
+        ffi.cursorModel.isInRemoteRect(_doubleFinerTapPosition);
+    if (isMobileMouseMode || isDesktopInRemoteRect) {
       inputModel.tap(MouseButtons.right);
     }
   }
@@ -245,9 +269,15 @@ class _RawTouchGestureDetectorRegionState
       if (ffi.cursorModel.shouldBlock(d.localPosition.dx, d.localPosition.dy)) {
         return;
       }
+      if (!ffi.cursorModel.isInRemoteRect(d.localPosition)) {
+        return;
+      }
+
+      _touchModePanStarted = true;
       if (isDesktop || isWebDesktop) {
         ffi.cursorModel.trySetRemoteWindowCoords();
       }
+
       // Workaround for the issue that the first pan event is sent a long time after the start event.
       // If the time interval between the start event and the first pan event is less than 500ms,
       // we consider to use the long press position as the start position.
@@ -280,10 +310,14 @@ class _RawTouchGestureDetectorRegionState
     if (ffi.cursorModel.shouldBlock(d.localPosition.dx, d.localPosition.dy)) {
       return;
     }
+    if (handleTouch && !_touchModePanStarted) {
+      return;
+    }
     ffi.cursorModel.updatePan(d.delta, d.localPosition, handleTouch);
   }
 
   onOneFingerPanEnd(DragEndDetails d) {
+    _touchModePanStarted = false;
     if (lastDeviceKind != PointerDeviceKind.touch) {
       return;
     }
diff --git a/flutter/lib/models/model.dart b/flutter/lib/models/model.dart
index 7765ebc40..616d80d11 100644
--- a/flutter/lib/models/model.dart
+++ b/flutter/lib/models/model.dart
@@ -9,6 +9,7 @@ import 'package:desktop_multi_window/desktop_multi_window.dart';
 import 'package:flutter/gestures.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter/services.dart';
+import 'package:flutter/widgets.dart';
 import 'package:flutter_hbb/common/widgets/peers_view.dart';
 import 'package:flutter_hbb/consts.dart';
 import 'package:flutter_hbb/models/ab_model.dart';
@@ -2040,16 +2041,56 @@ class CursorModel with ChangeNotifier {
       return false;
     }
     _lastIsBlocked = false;
-    moveLocal(x, y, adjust: parent.target?.canvasModel.getAdjustY() ?? 0);
+    if (!_moveLocalIfInRemoteRect(x, y)) {
+      return false;
+    }
     parent.target?.inputModel.moveMouse(_x, _y);
     return true;
   }
 
-  moveLocal(double x, double y, {double adjust = 0}) {
+  bool isInRemoteRect(Offset offset) {
+    return getRemotePosInRect(offset) != null;
+  }
+
+  Offset? getRemotePosInRect(Offset offset) {
+    final adjust = parent.target?.canvasModel.getAdjustY() ?? 0;
+    final newPos = _getNewPos(offset.dx, offset.dy, adjust);
+    final visibleRect = getVisibleRect();
+    if (!isPointInRect(newPos, visibleRect)) {
+      return null;
+    }
+    final rect = parent.target?.ffiModel.rect;
+    if (rect != null) {
+      if (!isPointInRect(newPos, rect)) {
+        return null;
+      }
+    }
+    return newPos;
+  }
+
+  Offset _getNewPos(double x, double y, double adjust) {
     final xoffset = parent.target?.canvasModel.x ?? 0;
     final yoffset = parent.target?.canvasModel.y ?? 0;
-    _x = (x - xoffset) / scale + _displayOriginX;
-    _y = (y - yoffset - adjust) / scale + _displayOriginY;
+    final newX = (x - xoffset) / scale + _displayOriginX;
+    final newY = (y - yoffset - adjust) / scale + _displayOriginY;
+    return Offset(newX, newY);
+  }
+
+  bool _moveLocalIfInRemoteRect(double x, double y) {
+    final newPos = getRemotePosInRect(Offset(x, y));
+    if (newPos == null) {
+      return false;
+    }
+    _x = newPos.dx;
+    _y = newPos.dy;
+    notifyListeners();
+    return true;
+  }
+
+  moveLocal(double x, double y, {double adjust = 0}) {
+    final newPos = _getNewPos(x, y, adjust);
+    _x = newPos.dx;
+    _y = newPos.dy;
     notifyListeners();
   }
 
@@ -2182,9 +2223,44 @@ class CursorModel with ChangeNotifier {
       }
     }
     if (!isMoved) {
+      final rect = parent.target?.ffiModel.rect;
+      if (rect == null) {
+        // unreachable
+        return;
+      }
+
+      Offset? movementInRect(double x, double y, Rect r) {
+        final isXInRect = x >= r.left && x <= r.right;
+        final isYInRect = y >= r.top && y <= r.bottom;
+        if (!(isXInRect || isYInRect)) {
+          return null;
+        }
+        if (x < r.left) {
+          x = r.left;
+        } else if (x > r.right) {
+          x = r.right;
+        }
+        if (y < r.top) {
+          y = r.top;
+        } else if (y > r.bottom) {
+          y = r.bottom;
+        }
+        return Offset(x, y);
+      }
+
       final scale = parent.target?.canvasModel.scale ?? 1.0;
-      _x += delta.dx / scale;
-      _y += delta.dy / scale;
+      var movement =
+          movementInRect(_x + delta.dx / scale, _y + delta.dy / scale, rect);
+      if (movement == null) {
+        return;
+      }
+      movement = movementInRect(movement.dx, movement.dy, getVisibleRect());
+      if (movement == null) {
+        return;
+      }
+
+      _x = movement.dx;
+      _y = movement.dy;
       parent.target?.inputModel.moveMouse(_x, _y);
     }
     notifyListeners();